From 2ceaf59767a8c1fb4530a196a23f586c09d84c94 Mon Sep 17 00:00:00 2001 From: Jure Bajic Date: Tue, 19 Jul 2022 12:28:19 +0200 Subject: [PATCH] Create query engine v2 (#444) Create version v2 of the query engine. Adjust CMake and lisp files Connect query engine v2 with storage engine v3 --- .gitignore | 7 + src/CMakeLists.txt | 1 + src/query/v2/CMakeLists.txt | 104 + src/query/v2/auth_checker.hpp | 29 + src/query/v2/common.cpp | 76 + src/query/v2/common.hpp | 111 + src/query/v2/config.hpp | 32 + src/query/v2/constants.hpp | 19 + src/query/v2/context.hpp | 89 + src/query/v2/cypher_query_interpreter.cpp | 158 + src/query/v2/cypher_query_interpreter.hpp | 152 + src/query/v2/db_accessor.hpp | 384 ++ src/query/v2/discard_value_stream.hpp | 24 + src/query/v2/dump.cpp | 541 +++ src/query/v2/dump.hpp | 66 + src/query/v2/exceptions.hpp | 227 + src/query/v2/frontend/ast/ast.lcp | 2676 +++++++++++ src/query/v2/frontend/ast/ast_visitor.hpp | 133 + .../v2/frontend/ast/cypher_main_visitor.cpp | 2362 ++++++++++ .../v2/frontend/ast/cypher_main_visitor.hpp | 886 ++++ src/query/v2/frontend/ast/pretty_print.cpp | 311 ++ src/query/v2/frontend/ast/pretty_print.hpp | 23 + .../v2/frontend/opencypher/grammar/Cypher.g4 | 391 ++ .../opencypher/grammar/CypherLexer.g4 | 208 + .../opencypher/grammar/MemgraphCypher.g4 | 376 ++ .../opencypher/grammar/MemgraphCypherLexer.g4 | 116 + .../opencypher/grammar/UnicodeCategories.g4 | 15 + src/query/v2/frontend/opencypher/parser.hpp | 68 + src/query/v2/frontend/parsing.cpp | 184 + src/query/v2/frontend/parsing.hpp | 27 + .../frontend/semantic/required_privileges.cpp | 152 + .../frontend/semantic/required_privileges.hpp | 18 + src/query/v2/frontend/semantic/symbol.lcp | 89 + .../v2/frontend/semantic/symbol_generator.cpp | 625 +++ .../v2/frontend/semantic/symbol_generator.hpp | 176 + .../v2/frontend/semantic/symbol_table.hpp | 64 + src/query/v2/frontend/stripped.cpp | 535 +++ src/query/v2/frontend/stripped.hpp | 103 + .../v2/frontend/stripped_lexer_constants.hpp | 2924 ++++++++++++ .../interpret/awesome_memgraph_functions.cpp | 1323 ++++++ .../interpret/awesome_memgraph_functions.hpp | 50 + src/query/v2/interpret/eval.cpp | 35 + src/query/v2/interpret/eval.hpp | 764 +++ src/query/v2/interpret/frame.hpp | 45 + src/query/v2/interpreter.cpp | 2412 ++++++++++ src/query/v2/interpreter.hpp | 436 ++ src/query/v2/metadata.cpp | 117 + src/query/v2/metadata.hpp | 90 + src/query/v2/parameters.hpp | 71 + src/query/v2/path.hpp | 146 + src/query/v2/plan/cost_estimator.hpp | 267 ++ src/query/v2/plan/operator.cpp | 4081 +++++++++++++++++ src/query/v2/plan/operator.lcp | 2305 ++++++++++ src/query/v2/plan/planner.hpp | 158 + src/query/v2/plan/preprocess.cpp | 599 +++ src/query/v2/plan/preprocess.hpp | 360 ++ src/query/v2/plan/pretty_print.cpp | 910 ++++ src/query/v2/plan/pretty_print.hpp | 230 + src/query/v2/plan/profile.cpp | 166 + src/query/v2/plan/profile.hpp | 47 + src/query/v2/plan/read_write_type_checker.cpp | 128 + src/query/v2/plan/read_write_type_checker.hpp | 95 + src/query/v2/plan/rewrite/index_lookup.cpp | 50 + src/query/v2/plan/rewrite/index_lookup.hpp | 668 +++ src/query/v2/plan/rule_based_planner.cpp | 594 +++ src/query/v2/plan/rule_based_planner.hpp | 561 +++ src/query/v2/plan/scoped_profile.hpp | 79 + src/query/v2/plan/variable_start_planner.cpp | 296 ++ src/query/v2/plan/variable_start_planner.hpp | 336 ++ src/query/v2/plan/vertex_count_cache.hpp | 141 + src/query/v2/procedure/cypher_type_ptr.hpp | 20 + src/query/v2/procedure/cypher_types.hpp | 293 ++ .../v2/procedure/mg_procedure_helpers.cpp | 36 + .../v2/procedure/mg_procedure_helpers.hpp | 69 + src/query/v2/procedure/mg_procedure_impl.cpp | 2798 +++++++++++ src/query/v2/procedure/mg_procedure_impl.hpp | 926 ++++ src/query/v2/procedure/module.cpp | 1258 +++++ src/query/v2/procedure/module.hpp | 246 + src/query/v2/procedure/py_module.cpp | 2649 +++++++++++ src/query/v2/procedure/py_module.hpp | 82 + src/query/v2/serialization/property_value.cpp | 127 + src/query/v2/serialization/property_value.hpp | 32 + src/query/v2/stream.hpp | 63 + src/query/v2/stream/common.cpp | 45 + src/query/v2/stream/common.hpp | 87 + src/query/v2/stream/sources.cpp | 137 + src/query/v2/stream/sources.hpp | 95 + src/query/v2/stream/streams.cpp | 772 ++++ src/query/v2/stream/streams.hpp | 206 + src/query/v2/trigger.cpp | 441 ++ src/query/v2/trigger.hpp | 119 + src/query/v2/trigger_context.cpp | 557 +++ src/query/v2/trigger_context.hpp | 365 ++ src/query/v2/typed_value.cpp | 1108 +++++ src/query/v2/typed_value.hpp | 739 +++ 95 files changed, 45312 insertions(+) create mode 100644 src/query/v2/CMakeLists.txt create mode 100644 src/query/v2/auth_checker.hpp create mode 100644 src/query/v2/common.cpp create mode 100644 src/query/v2/common.hpp create mode 100644 src/query/v2/config.hpp create mode 100644 src/query/v2/constants.hpp create mode 100644 src/query/v2/context.hpp create mode 100644 src/query/v2/cypher_query_interpreter.cpp create mode 100644 src/query/v2/cypher_query_interpreter.hpp create mode 100644 src/query/v2/db_accessor.hpp create mode 100644 src/query/v2/discard_value_stream.hpp create mode 100644 src/query/v2/dump.cpp create mode 100644 src/query/v2/dump.hpp create mode 100644 src/query/v2/exceptions.hpp create mode 100644 src/query/v2/frontend/ast/ast.lcp create mode 100644 src/query/v2/frontend/ast/ast_visitor.hpp create mode 100644 src/query/v2/frontend/ast/cypher_main_visitor.cpp create mode 100644 src/query/v2/frontend/ast/cypher_main_visitor.hpp create mode 100644 src/query/v2/frontend/ast/pretty_print.cpp create mode 100644 src/query/v2/frontend/ast/pretty_print.hpp create mode 100644 src/query/v2/frontend/opencypher/grammar/Cypher.g4 create mode 100644 src/query/v2/frontend/opencypher/grammar/CypherLexer.g4 create mode 100644 src/query/v2/frontend/opencypher/grammar/MemgraphCypher.g4 create mode 100644 src/query/v2/frontend/opencypher/grammar/MemgraphCypherLexer.g4 create mode 100644 src/query/v2/frontend/opencypher/grammar/UnicodeCategories.g4 create mode 100644 src/query/v2/frontend/opencypher/parser.hpp create mode 100644 src/query/v2/frontend/parsing.cpp create mode 100644 src/query/v2/frontend/parsing.hpp create mode 100644 src/query/v2/frontend/semantic/required_privileges.cpp create mode 100644 src/query/v2/frontend/semantic/required_privileges.hpp create mode 100644 src/query/v2/frontend/semantic/symbol.lcp create mode 100644 src/query/v2/frontend/semantic/symbol_generator.cpp create mode 100644 src/query/v2/frontend/semantic/symbol_generator.hpp create mode 100644 src/query/v2/frontend/semantic/symbol_table.hpp create mode 100644 src/query/v2/frontend/stripped.cpp create mode 100644 src/query/v2/frontend/stripped.hpp create mode 100644 src/query/v2/frontend/stripped_lexer_constants.hpp create mode 100644 src/query/v2/interpret/awesome_memgraph_functions.cpp create mode 100644 src/query/v2/interpret/awesome_memgraph_functions.hpp create mode 100644 src/query/v2/interpret/eval.cpp create mode 100644 src/query/v2/interpret/eval.hpp create mode 100644 src/query/v2/interpret/frame.hpp create mode 100644 src/query/v2/interpreter.cpp create mode 100644 src/query/v2/interpreter.hpp create mode 100644 src/query/v2/metadata.cpp create mode 100644 src/query/v2/metadata.hpp create mode 100644 src/query/v2/parameters.hpp create mode 100644 src/query/v2/path.hpp create mode 100644 src/query/v2/plan/cost_estimator.hpp create mode 100644 src/query/v2/plan/operator.cpp create mode 100644 src/query/v2/plan/operator.lcp create mode 100644 src/query/v2/plan/planner.hpp create mode 100644 src/query/v2/plan/preprocess.cpp create mode 100644 src/query/v2/plan/preprocess.hpp create mode 100644 src/query/v2/plan/pretty_print.cpp create mode 100644 src/query/v2/plan/pretty_print.hpp create mode 100644 src/query/v2/plan/profile.cpp create mode 100644 src/query/v2/plan/profile.hpp create mode 100644 src/query/v2/plan/read_write_type_checker.cpp create mode 100644 src/query/v2/plan/read_write_type_checker.hpp create mode 100644 src/query/v2/plan/rewrite/index_lookup.cpp create mode 100644 src/query/v2/plan/rewrite/index_lookup.hpp create mode 100644 src/query/v2/plan/rule_based_planner.cpp create mode 100644 src/query/v2/plan/rule_based_planner.hpp create mode 100644 src/query/v2/plan/scoped_profile.hpp create mode 100644 src/query/v2/plan/variable_start_planner.cpp create mode 100644 src/query/v2/plan/variable_start_planner.hpp create mode 100644 src/query/v2/plan/vertex_count_cache.hpp create mode 100644 src/query/v2/procedure/cypher_type_ptr.hpp create mode 100644 src/query/v2/procedure/cypher_types.hpp create mode 100644 src/query/v2/procedure/mg_procedure_helpers.cpp create mode 100644 src/query/v2/procedure/mg_procedure_helpers.hpp create mode 100644 src/query/v2/procedure/mg_procedure_impl.cpp create mode 100644 src/query/v2/procedure/mg_procedure_impl.hpp create mode 100644 src/query/v2/procedure/module.cpp create mode 100644 src/query/v2/procedure/module.hpp create mode 100644 src/query/v2/procedure/py_module.cpp create mode 100644 src/query/v2/procedure/py_module.hpp create mode 100644 src/query/v2/serialization/property_value.cpp create mode 100644 src/query/v2/serialization/property_value.hpp create mode 100644 src/query/v2/stream.hpp create mode 100644 src/query/v2/stream/common.cpp create mode 100644 src/query/v2/stream/common.hpp create mode 100644 src/query/v2/stream/sources.cpp create mode 100644 src/query/v2/stream/sources.hpp create mode 100644 src/query/v2/stream/streams.cpp create mode 100644 src/query/v2/stream/streams.hpp create mode 100644 src/query/v2/trigger.cpp create mode 100644 src/query/v2/trigger.hpp create mode 100644 src/query/v2/trigger_context.cpp create mode 100644 src/query/v2/trigger_context.hpp create mode 100644 src/query/v2/typed_value.cpp create mode 100644 src/query/v2/typed_value.hpp diff --git a/.gitignore b/.gitignore index e1a4187b0..8dd3dfb0f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ cmake-build-* cmake/DownloadProject/ dist/ src/query/frontend/opencypher/generated/ +src/query/v2/frontend/opencypher/generated/ tags ve/ ve3/ @@ -50,15 +51,21 @@ src/distributed/pull_produce_rpc_messages.hpp src/distributed/storage_gc_rpc_messages.hpp src/distributed/token_sharing_rpc_messages.hpp src/distributed/updates_rpc_messages.hpp +src/query/v2/frontend/ast/ast.hpp src/query/frontend/ast/ast.hpp src/query/distributed/frontend/ast/ast_serialization.hpp +src/query/v2/distributed/frontend/ast/ast_serialization.hpp src/durability/distributed/state_delta.hpp src/durability/single_node/state_delta.hpp src/durability/single_node_ha/state_delta.hpp src/query/frontend/semantic/symbol.hpp +src/query/v2/frontend/semantic/symbol.hpp src/query/distributed/frontend/semantic/symbol_serialization.hpp +src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp src/query/distributed/plan/ops.hpp +src/query/v2/distributed/plan/ops.hpp src/query/plan/operator.hpp +src/query/v2/plan/operator.hpp src/raft/log_entry.hpp src/raft/raft_rpc_messages.hpp src/raft/snapshot_metadata.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e6160d972..efc653b9a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,6 +13,7 @@ add_subdirectory(storage/v2) add_subdirectory(storage/v3) add_subdirectory(integrations) add_subdirectory(query) +add_subdirectory(query/v2) add_subdirectory(slk) add_subdirectory(rpc) add_subdirectory(auth) diff --git a/src/query/v2/CMakeLists.txt b/src/query/v2/CMakeLists.txt new file mode 100644 index 000000000..3efb0c91e --- /dev/null +++ b/src/query/v2/CMakeLists.txt @@ -0,0 +1,104 @@ +define_add_lcp(add_lcp_query lcp_query_v2_cpp_files generated_lcp_query_v2_files) + +add_lcp_query(frontend/ast/ast.lcp) +add_lcp_query(frontend/semantic/symbol.lcp) +add_lcp_query(plan/operator.lcp) + +add_custom_target(generate_lcp_query_v2 DEPENDS ${generated_lcp_query_v2_files}) + +set(mg_query_v2_sources + ${lcp_query_v2_cpp_files} + common.cpp + cypher_query_interpreter.cpp + dump.cpp + frontend/ast/cypher_main_visitor.cpp + frontend/ast/pretty_print.cpp + frontend/parsing.cpp + frontend/semantic/required_privileges.cpp + frontend/semantic/symbol_generator.cpp + frontend/stripped.cpp + interpret/awesome_memgraph_functions.cpp + interpret/eval.cpp + interpreter.cpp + metadata.cpp + plan/operator.cpp + plan/preprocess.cpp + plan/pretty_print.cpp + plan/profile.cpp + plan/read_write_type_checker.cpp + plan/rewrite/index_lookup.cpp + plan/rule_based_planner.cpp + plan/variable_start_planner.cpp + procedure/mg_procedure_impl.cpp + procedure/mg_procedure_helpers.cpp + procedure/module.cpp + procedure/py_module.cpp + serialization/property_value.cpp + stream/streams.cpp + stream/sources.cpp + stream/common.cpp + trigger.cpp + trigger_context.cpp + typed_value.cpp) + +find_package(Boost REQUIRED) + +add_library(mg-query-v2 STATIC ${mg_query_v2_sources}) +add_dependencies(mg-query-v2 generate_lcp_query_v2) +target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include) +target_link_libraries(mg-query-v2 dl cppitertools Boost::headers) +target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory) + +if(NOT "${MG_PYTHON_PATH}" STREQUAL "") + set(Python3_ROOT_DIR "${MG_PYTHON_PATH}") +endif() + +if("${MG_PYTHON_VERSION}" STREQUAL "") + find_package(Python3 3.5 REQUIRED COMPONENTS Development) +else() + find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development) +endif() + +target_link_libraries(mg-query-v2 Python3::Python) + +# Generate Antlr openCypher parser +set(opencypher_frontend ${CMAKE_CURRENT_SOURCE_DIR}/frontend/opencypher) +set(opencypher_generated ${opencypher_frontend}/generated) +set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4) +set(opencypher_parser_grammar ${opencypher_frontend}/grammar/MemgraphCypher.g4) + +set(antlr_opencypher_generated_src + ${opencypher_generated}/MemgraphCypherLexer.cpp + ${opencypher_generated}/MemgraphCypher.cpp + ${opencypher_generated}/MemgraphCypherBaseVisitor.cpp + ${opencypher_generated}/MemgraphCypherVisitor.cpp +) +set(antlr_opencypher_generated_include + ${opencypher_generated}/MemgraphCypherLexer.h + ${opencypher_generated}/MemgraphCypher.h + ${opencypher_generated}/MemgraphCypherBaseVisitor.h + ${opencypher_generated}/MemgraphCypherVisitor.h +) + +add_custom_command( + OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include} + COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated} + COMMAND + java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.9.2-complete.jar + -Dlanguage=Cpp -visitor -package antlropencypher + -o ${opencypher_generated} + ${opencypher_lexer_grammar} ${opencypher_parser_grammar} + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" + DEPENDS + ${opencypher_lexer_grammar} ${opencypher_parser_grammar} + ${opencypher_frontend}/grammar/CypherLexer.g4 + ${opencypher_frontend}/grammar/Cypher.g4) + +add_custom_target(generate_opencypher_parser_v2 + DEPENDS ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}) + +add_library(antlr_opencypher_parser_lib_v2 STATIC ${antlr_opencypher_generated_src}) +add_dependencies(antlr_opencypher_parser_lib_v2 generate_opencypher_parser_v2) +target_link_libraries(antlr_opencypher_parser_lib_v2 antlr4) + +target_link_libraries(mg-query-v2 antlr_opencypher_parser_lib_v2) diff --git a/src/query/v2/auth_checker.hpp b/src/query/v2/auth_checker.hpp new file mode 100644 index 000000000..48d755a16 --- /dev/null +++ b/src/query/v2/auth_checker.hpp @@ -0,0 +1,29 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "query/v2/frontend/ast/ast.hpp" + +namespace memgraph::query::v2 { +class AuthChecker { + public: + virtual bool IsUserAuthorized(const std::optional &username, + const std::vector &privileges) const = 0; +}; + +class AllowEverythingAuthChecker final : public query::v2::AuthChecker { + bool IsUserAuthorized(const std::optional & /*username*/, + const std::vector & /*privileges*/) const override { + return true; + } +}; +} // namespace memgraph::query::v2 diff --git a/src/query/v2/common.cpp b/src/query/v2/common.cpp new file mode 100644 index 000000000..4ca63f6b0 --- /dev/null +++ b/src/query/v2/common.cpp @@ -0,0 +1,76 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/common.hpp" + +namespace memgraph::query::v2 { + +namespace impl { + +bool TypedValueCompare(const TypedValue &a, const TypedValue &b) { + // in ordering null comes after everything else + // at the same time Null is not less that null + // first deal with Null < Whatever case + if (a.IsNull()) return false; + // now deal with NotNull < Null case + if (b.IsNull()) return true; + + // comparisons are from this point legal only between values of + // the same type, or int+float combinations + if ((a.type() != b.type() && !(a.IsNumeric() && b.IsNumeric()))) + throw QueryRuntimeException("Can't compare value of type {} to value of type {}.", a.type(), b.type()); + + switch (a.type()) { + case TypedValue::Type::Bool: + return !a.ValueBool() && b.ValueBool(); + case TypedValue::Type::Int: + if (b.type() == TypedValue::Type::Double) + return a.ValueInt() < b.ValueDouble(); + else + return a.ValueInt() < b.ValueInt(); + case TypedValue::Type::Double: + if (b.type() == TypedValue::Type::Int) + return a.ValueDouble() < b.ValueInt(); + else + return a.ValueDouble() < b.ValueDouble(); + case TypedValue::Type::String: + // NOLINTNEXTLINE(modernize-use-nullptr) + return a.ValueString() < b.ValueString(); + case TypedValue::Type::Date: + // NOLINTNEXTLINE(modernize-use-nullptr) + return a.ValueDate() < b.ValueDate(); + case TypedValue::Type::LocalTime: + // NOLINTNEXTLINE(modernize-use-nullptr) + return a.ValueLocalTime() < b.ValueLocalTime(); + case TypedValue::Type::LocalDateTime: + // NOLINTNEXTLINE(modernize-use-nullptr) + return a.ValueLocalDateTime() < b.ValueLocalDateTime(); + case TypedValue::Type::Duration: + // NOLINTNEXTLINE(modernize-use-nullptr) + return a.ValueDuration() < b.ValueDuration(); + case TypedValue::Type::List: + case TypedValue::Type::Map: + case TypedValue::Type::Vertex: + case TypedValue::Type::Edge: + case TypedValue::Type::Path: + throw QueryRuntimeException("Comparison is not defined for values of type {}.", a.type()); + case TypedValue::Type::Null: + LOG_FATAL("Invalid type"); + } +} + +} // namespace impl + +int64_t QueryTimestamp() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count(); +} +} // namespace memgraph::query::v2 diff --git a/src/query/v2/common.hpp b/src/query/v2/common.hpp new file mode 100644 index 000000000..e79ca996c --- /dev/null +++ b/src/query/v2/common.hpp @@ -0,0 +1,111 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +/// @file +#pragma once + +#include +#include +#include +#include + +#include "query/v2/db_accessor.hpp" +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/semantic/symbol.hpp" +#include "query/v2/typed_value.hpp" +#include "storage/v3/id_types.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/view.hpp" +#include "utils/logging.hpp" + +namespace memgraph::query::v2 { + +namespace impl { +bool TypedValueCompare(const TypedValue &a, const TypedValue &b); +} // namespace impl + +/// Custom Comparator type for comparing vectors of TypedValues. +/// +/// Does lexicographical ordering of elements based on the above +/// defined TypedValueCompare, and also accepts a vector of Orderings +/// the define how respective elements compare. +class TypedValueVectorCompare final { + public: + TypedValueVectorCompare() {} + explicit TypedValueVectorCompare(const std::vector &ordering) : ordering_(ordering) {} + + template + bool operator()(const std::vector &c1, const std::vector &c2) const { + // ordering is invalid if there are more elements in the collections + // then there are in the ordering_ vector + MG_ASSERT(c1.size() <= ordering_.size() && c2.size() <= ordering_.size(), + "Collections contain more elements then there are orderings"); + + auto c1_it = c1.begin(); + auto c2_it = c2.begin(); + auto ordering_it = ordering_.begin(); + for (; c1_it != c1.end() && c2_it != c2.end(); c1_it++, c2_it++, ordering_it++) { + if (impl::TypedValueCompare(*c1_it, *c2_it)) return *ordering_it == Ordering::ASC; + if (impl::TypedValueCompare(*c2_it, *c1_it)) return *ordering_it == Ordering::DESC; + } + + // at least one collection is exhausted + // c1 is less then c2 iff c1 reached the end but c2 didn't + return (c1_it == c1.end()) && (c2_it != c2.end()); + } + + // TODO: Remove this, member is public + const auto &ordering() const { return ordering_; } + + std::vector ordering_; +}; + +/// Raise QueryRuntimeException if the value for symbol isn't of expected type. +inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue::Type expected) { + if (value.type() != expected) + throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type()); +} + +template +concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key, + const storage::v3::PropertyValue new_value) { + { accessor.SetProperty(key, new_value) } -> std::same_as>; +}; + +/// Set a property `value` mapped with given `key` on a `record`. +/// +/// @throw QueryRuntimeException if value cannot be set as a property value +template +storage::v3::PropertyValue PropsSetChecked(T *record, const storage::v3::PropertyId &key, const TypedValue &value) { + try { + auto maybe_old_value = record->SetProperty(key, storage::v3::PropertyValue(value)); + if (maybe_old_value.HasError()) { + switch (maybe_old_value.GetError()) { + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to set properties on a deleted object."); + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Can't set property because properties on edges are disabled."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when setting a property."); + } + } + return std::move(*maybe_old_value); + } catch (const TypedValueException &) { + throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type()); + } +} + +int64_t QueryTimestamp(); +} // namespace memgraph::query::v2 diff --git a/src/query/v2/config.hpp b/src/query/v2/config.hpp new file mode 100644 index 000000000..13b0539cc --- /dev/null +++ b/src/query/v2/config.hpp @@ -0,0 +1,32 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once +#include +#include + +namespace memgraph::query::v2 { +struct InterpreterConfig { + struct Query { + bool allow_load_csv{true}; + } query; + + // The default execution timeout is 10 minutes. + double execution_timeout_sec{600.0}; + // The same as \ref memgraph::storage::v3::replication::ReplicationClientConfig + std::chrono::seconds replication_replica_check_frequency{1}; + + std::string default_kafka_bootstrap_servers; + std::string default_pulsar_service_url; + uint32_t stream_transaction_conflict_retries; + std::chrono::milliseconds stream_transaction_retry_interval; +}; +} // namespace memgraph::query::v2 diff --git a/src/query/v2/constants.hpp b/src/query/v2/constants.hpp new file mode 100644 index 000000000..c19d939b6 --- /dev/null +++ b/src/query/v2/constants.hpp @@ -0,0 +1,19 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once +#include +#include + +namespace memgraph::query::v2 { +inline constexpr uint16_t kDefaultReplicationPort = 10000; +inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0"; +} // namespace memgraph::query::v2 diff --git a/src/query/v2/context.hpp b/src/query/v2/context.hpp new file mode 100644 index 000000000..982de53a5 --- /dev/null +++ b/src/query/v2/context.hpp @@ -0,0 +1,89 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "query/v2/common.hpp" +#include "query/v2/frontend/semantic/symbol_table.hpp" +#include "query/v2/metadata.hpp" +#include "query/v2/parameters.hpp" +#include "query/v2/plan/profile.hpp" +#include "query/v2/trigger.hpp" +#include "utils/async_timer.hpp" + +namespace memgraph::query::v2 { + +struct EvaluationContext { + /// Memory for allocations during evaluation of a *single* Pull call. + /// + /// Although the assigned memory may live longer than the duration of a Pull + /// (e.g. memory is the same as the whole execution memory), you have to treat + /// it as if the lifetime is only valid during the Pull. + utils::MemoryResource *memory{utils::NewDeleteResource()}; + int64_t timestamp{-1}; + Parameters parameters; + /// All properties indexable via PropertyIx + std::vector properties; + /// All labels indexable via LabelIx + std::vector labels; + /// All counters generated by `counter` function, mutable because the function + /// modifies the values + mutable std::unordered_map counters; +}; + +inline std::vector NamesToProperties(const std::vector &property_names, + DbAccessor *dba) { + std::vector properties; + properties.reserve(property_names.size()); + for (const auto &name : property_names) { + properties.push_back(dba->NameToProperty(name)); + } + return properties; +} + +inline std::vector NamesToLabels(const std::vector &label_names, DbAccessor *dba) { + std::vector labels; + labels.reserve(label_names.size()); + for (const auto &name : label_names) { + labels.push_back(dba->NameToLabel(name)); + } + return labels; +} + +struct ExecutionContext { + DbAccessor *db_accessor{nullptr}; + SymbolTable symbol_table; + EvaluationContext evaluation_context; + std::atomic *is_shutting_down{nullptr}; + bool is_profile_query{false}; + std::chrono::duration profile_execution_time; + plan::ProfilingStats stats; + plan::ProfilingStats *stats_root{nullptr}; + ExecutionStats execution_stats; + TriggerContextCollector *trigger_context_collector{nullptr}; + utils::AsyncTimer timer; +}; + +static_assert(std::is_move_assignable_v, "ExecutionContext must be move assignable!"); +static_assert(std::is_move_constructible_v, "ExecutionContext must be move constructible!"); + +inline bool MustAbort(const ExecutionContext &context) noexcept { + return (context.is_shutting_down != nullptr && context.is_shutting_down->load(std::memory_order_acquire)) || + context.timer.IsExpired(); +} + +inline plan::ProfilingStatsWithTotalTime GetStatsWithTotalTime(const ExecutionContext &context) { + return plan::ProfilingStatsWithTotalTime{context.stats, context.profile_execution_time}; +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/cypher_query_interpreter.cpp b/src/query/v2/cypher_query_interpreter.cpp new file mode 100644 index 000000000..42e2b3cf3 --- /dev/null +++ b/src/query/v2/cypher_query_interpreter.cpp @@ -0,0 +1,158 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/cypher_query_interpreter.hpp" + +// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) +DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner."); +// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) +DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.", + FLAG_IN_RANGE(0, std::numeric_limits::max())); + +namespace memgraph::query::v2 { +CachedPlan::CachedPlan(std::unique_ptr plan) : plan_(std::move(plan)) {} + +ParsedQuery ParseQuery(const std::string &query_string, const std::map ¶ms, + utils::SkipList *cache, utils::SpinLock *antlr_lock, + const InterpreterConfig::Query &query_config) { + // Strip the query for caching purposes. The process of stripping a query + // "normalizes" it by replacing any literals with new parameters. This + // results in just the *structure* of the query being taken into account for + // caching. + frontend::StrippedQuery stripped_query{query_string}; + + // Copy over the parameters that were introduced during stripping. + Parameters parameters{stripped_query.literals()}; + + // Check that all user-specified parameters are provided. + for (const auto ¶m_pair : stripped_query.parameters()) { + auto it = params.find(param_pair.second); + + if (it == params.end()) { + throw query::v2::UnprovidedParameterError("Parameter ${} not provided.", param_pair.second); + } + + parameters.Add(param_pair.first, it->second); + } + + // Cache the query's AST if it isn't already. + auto hash = stripped_query.hash(); + auto accessor = cache->access(); + auto it = accessor.find(hash); + std::unique_ptr parser; + + // Return a copy of both the AST storage and the query. + CachedQuery result; + bool is_cacheable = true; + + auto get_information_from_cache = [&](const auto &cached_query) { + result.ast_storage.properties_ = cached_query.ast_storage.properties_; + result.ast_storage.labels_ = cached_query.ast_storage.labels_; + result.ast_storage.edge_types_ = cached_query.ast_storage.edge_types_; + + result.query = cached_query.query->Clone(&result.ast_storage); + result.required_privileges = cached_query.required_privileges; + }; + + if (it == accessor.end()) { + { + std::unique_lock guard(*antlr_lock); + + try { + parser = std::make_unique(stripped_query.query()); + } catch (const SyntaxException &e) { + // There is a syntax exception in the stripped query. Re-run the parser + // on the original query to get an appropriate error messsage. + parser = std::make_unique(query_string); + + // If an exception was not thrown here, the stripper messed something + // up. + LOG_FATAL("The stripped query can't be parsed, but the original can."); + } + } + + // Convert the ANTLR4 parse tree into an AST. + AstStorage ast_storage; + frontend::ParsingContext context{true}; + frontend::CypherMainVisitor visitor(context, &ast_storage); + + visitor.visit(parser->tree()); + + if (visitor.GetQueryInfo().has_load_csv && !query_config.allow_load_csv) { + throw utils::BasicException("Load CSV not allowed on this instance because it was disabled by a config."); + } + + if (visitor.GetQueryInfo().is_cacheable) { + CachedQuery cached_query{std::move(ast_storage), visitor.query(), + query::v2::GetRequiredPrivileges(visitor.query())}; + it = accessor.insert({hash, std::move(cached_query)}).first; + + get_information_from_cache(it->second); + } else { + result.ast_storage.properties_ = ast_storage.properties_; + result.ast_storage.labels_ = ast_storage.labels_; + result.ast_storage.edge_types_ = ast_storage.edge_types_; + + result.query = visitor.query()->Clone(&result.ast_storage); + result.required_privileges = query::v2::GetRequiredPrivileges(visitor.query()); + + is_cacheable = false; + } + } else { + get_information_from_cache(it->second); + } + + return ParsedQuery{query_string, + params, + std::move(parameters), + std::move(stripped_query), + std::move(result.ast_storage), + result.query, + std::move(result.required_privileges), + is_cacheable}; +} + +std::unique_ptr MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters, + DbAccessor *db_accessor, + const std::vector &predefined_identifiers) { + auto vertex_counts = plan::MakeVertexCountCache(db_accessor); + auto symbol_table = MakeSymbolTable(query, predefined_identifiers); + auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts); + auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner); + return std::make_unique(std::move(root), cost, std::move(ast_storage), + std::move(symbol_table)); +} + +std::shared_ptr CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query, + const Parameters ¶meters, utils::SkipList *plan_cache, + DbAccessor *db_accessor, + const std::vector &predefined_identifiers) { + std::optional::Accessor> plan_cache_access; + if (plan_cache) { + plan_cache_access.emplace(plan_cache->access()); + auto it = plan_cache_access->find(hash); + if (it != plan_cache_access->end()) { + if (it->second->IsExpired()) { + plan_cache_access->remove(hash); + } else { + return it->second; + } + } + } + + auto plan = std::make_shared( + MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers)); + if (plan_cache_access) { + plan_cache_access->insert({hash, plan}); + } + return plan; +} +} // namespace memgraph::query::v2 diff --git a/src/query/v2/cypher_query_interpreter.hpp b/src/query/v2/cypher_query_interpreter.hpp new file mode 100644 index 000000000..423eafdde --- /dev/null +++ b/src/query/v2/cypher_query_interpreter.hpp @@ -0,0 +1,152 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "query/v2/config.hpp" +#include "query/v2/frontend/ast/cypher_main_visitor.hpp" +#include "query/v2/frontend/opencypher/parser.hpp" +#include "query/v2/frontend/semantic/required_privileges.hpp" +#include "query/v2/frontend/semantic/symbol_generator.hpp" +#include "query/v2/frontend/stripped.hpp" +#include "query/v2/plan/planner.hpp" +#include "utils/flag_validation.hpp" +#include "utils/timer.hpp" + +// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) +DECLARE_bool(query_cost_planner); +// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) +DECLARE_int32(query_plan_cache_ttl); + +namespace memgraph::query::v2 { + +// TODO: Maybe this should move to query/plan/planner. +/// Interface for accessing the root operator of a logical plan. +class LogicalPlan { + public: + explicit LogicalPlan() = default; + + virtual ~LogicalPlan() = default; + + LogicalPlan(const LogicalPlan &) = default; + LogicalPlan &operator=(const LogicalPlan &) = default; + LogicalPlan(LogicalPlan &&) = default; + LogicalPlan &operator=(LogicalPlan &&) = default; + + virtual const plan::LogicalOperator &GetRoot() const = 0; + virtual double GetCost() const = 0; + virtual const SymbolTable &GetSymbolTable() const = 0; + virtual const AstStorage &GetAstStorage() const = 0; +}; + +class CachedPlan { + public: + explicit CachedPlan(std::unique_ptr plan); + + const auto &plan() const { return plan_->GetRoot(); } + double cost() const { return plan_->GetCost(); } + const auto &symbol_table() const { return plan_->GetSymbolTable(); } + const auto &ast_storage() const { return plan_->GetAstStorage(); } + + bool IsExpired() const { + // NOLINTNEXTLINE (modernize-use-nullptr) + return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl); + }; + + private: + std::unique_ptr plan_; + utils::Timer cache_timer_; +}; + +struct CachedQuery { + AstStorage ast_storage; + Query *query; + std::vector required_privileges; +}; + +struct QueryCacheEntry { + bool operator==(const QueryCacheEntry &other) const { return first == other.first; } + bool operator<(const QueryCacheEntry &other) const { return first < other.first; } + bool operator==(const uint64_t &other) const { return first == other; } + bool operator<(const uint64_t &other) const { return first < other; } + + uint64_t first; + // TODO: Maybe store the query string here and use it as a key with the hash + // so that we eliminate the risk of hash collisions. + CachedQuery second; +}; + +struct PlanCacheEntry { + bool operator==(const PlanCacheEntry &other) const { return first == other.first; } + bool operator<(const PlanCacheEntry &other) const { return first < other.first; } + bool operator==(const uint64_t &other) const { return first == other; } + bool operator<(const uint64_t &other) const { return first < other; } + + uint64_t first; + // TODO: Maybe store the query string here and use it as a key with the hash + // so that we eliminate the risk of hash collisions. + std::shared_ptr second; +}; + +/** + * A container for data related to the parsing of a query. + */ +struct ParsedQuery { + std::string query_string; + std::map user_parameters; + Parameters parameters; + frontend::StrippedQuery stripped_query; + AstStorage ast_storage; + Query *query; + std::vector required_privileges; + bool is_cacheable{true}; +}; + +ParsedQuery ParseQuery(const std::string &query_string, const std::map ¶ms, + utils::SkipList *cache, utils::SpinLock *antlr_lock, + const InterpreterConfig::Query &query_config); + +class SingleNodeLogicalPlan final : public LogicalPlan { + public: + SingleNodeLogicalPlan(std::unique_ptr root, double cost, AstStorage storage, + const SymbolTable &symbol_table) + : root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(symbol_table) {} + + const plan::LogicalOperator &GetRoot() const override { return *root_; } + double GetCost() const override { return cost_; } + const SymbolTable &GetSymbolTable() const override { return symbol_table_; } + const AstStorage &GetAstStorage() const override { return storage_; } + + private: + std::unique_ptr root_; + double cost_; + AstStorage storage_; + SymbolTable symbol_table_; +}; + +std::unique_ptr MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters, + DbAccessor *db_accessor, + const std::vector &predefined_identifiers); + +/** + * Return the parsed *Cypher* query's AST cached logical plan, or create and + * cache a fresh one if it doesn't yet exist. + * @param predefined_identifiers optional identifiers you want to inject into a query. + * If an identifier is not defined in a scope, we check the predefined identifiers. + * If an identifier is contained there, we inject it at that place and remove it, + * because a predefined identifier can be used only in one scope. + */ +std::shared_ptr CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query, + const Parameters ¶meters, utils::SkipList *plan_cache, + DbAccessor *db_accessor, + const std::vector &predefined_identifiers = {}); + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/db_accessor.hpp b/src/query/v2/db_accessor.hpp new file mode 100644 index 000000000..90ea6d431 --- /dev/null +++ b/src/query/v2/db_accessor.hpp @@ -0,0 +1,384 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include +#include + +#include "query/v2/exceptions.hpp" +#include "storage/v3/id_types.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/result.hpp" + +/////////////////////////////////////////////////////////// +// Our communication layer and query engine don't mix +// very well on Centos because OpenSSL version avaialable +// on Centos 7 include libkrb5 which has brilliant macros +// called TRUE and FALSE. For more detailed explanation go +// to memgraph.cpp. +// +// Because of the replication storage now uses some form of +// communication so we have some unwanted macros. +// This cannot be avoided by simple include orderings so we +// simply undefine those macros as we're sure that libkrb5 +// won't and can't be used anywhere in the query engine. +#include "storage/v3/storage.hpp" + +#undef FALSE +#undef TRUE +/////////////////////////////////////////////////////////// + +#include "storage/v3/view.hpp" +#include "utils/bound.hpp" +#include "utils/exceptions.hpp" + +namespace memgraph::query::v2 { + +class VertexAccessor; + +class EdgeAccessor final { + public: + storage::v3::EdgeAccessor impl_; + + public: + explicit EdgeAccessor(storage::v3::EdgeAccessor impl) : impl_(std::move(impl)) {} + + bool IsVisible(storage::v3::View view) const { return impl_.IsVisible(view); } + + storage::v3::EdgeTypeId EdgeType() const { return impl_.EdgeType(); } + + auto Properties(storage::v3::View view) const { return impl_.Properties(view); } + + storage::v3::Result GetProperty(storage::v3::View view, + storage::v3::PropertyId key) const { + return impl_.GetProperty(key, view); + } + + storage::v3::Result SetProperty(storage::v3::PropertyId key, + const storage::v3::PropertyValue &value) { + return impl_.SetProperty(key, value); + } + + storage::v3::Result RemoveProperty(storage::v3::PropertyId key) { + return SetProperty(key, storage::v3::PropertyValue()); + } + + storage::v3::Result> ClearProperties() { + return impl_.ClearProperties(); + } + + VertexAccessor To() const; + + VertexAccessor From() const; + + bool IsCycle() const; + + int64_t CypherId() const { return impl_.Gid().AsInt(); } + + storage::v3::Gid Gid() const noexcept { return impl_.Gid(); } + + bool operator==(const EdgeAccessor &e) const noexcept { return impl_ == e.impl_; } + + bool operator!=(const EdgeAccessor &e) const noexcept { return !(*this == e); } +}; + +class VertexAccessor final { + public: + storage::v3::VertexAccessor impl_; + + static EdgeAccessor MakeEdgeAccessor(const storage::v3::EdgeAccessor impl) { return EdgeAccessor(impl); } + + public: + explicit VertexAccessor(storage::v3::VertexAccessor impl) : impl_(impl) {} + + bool IsVisible(storage::v3::View view) const { return impl_.IsVisible(view); } + + auto Labels(storage::v3::View view) const { return impl_.Labels(view); } + + storage::v3::Result AddLabel(storage::v3::LabelId label) { return impl_.AddLabel(label); } + + storage::v3::Result RemoveLabel(storage::v3::LabelId label) { return impl_.RemoveLabel(label); } + + storage::v3::Result HasLabel(storage::v3::View view, storage::v3::LabelId label) const { + return impl_.HasLabel(label, view); + } + + auto Properties(storage::v3::View view) const { return impl_.Properties(view); } + + storage::v3::Result GetProperty(storage::v3::View view, + storage::v3::PropertyId key) const { + return impl_.GetProperty(key, view); + } + + storage::v3::Result SetProperty(storage::v3::PropertyId key, + const storage::v3::PropertyValue &value) { + return impl_.SetProperty(key, value); + } + + storage::v3::Result RemoveProperty(storage::v3::PropertyId key) { + return SetProperty(key, storage::v3::PropertyValue()); + } + + storage::v3::Result> ClearProperties() { + return impl_.ClearProperties(); + } + + auto InEdges(storage::v3::View view, const std::vector &edge_types) const + -> storage::v3::Result { + auto maybe_edges = impl_.InEdges(view, edge_types); + if (maybe_edges.HasError()) return maybe_edges.GetError(); + return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges)); + } + + auto InEdges(storage::v3::View view) const { return InEdges(view, {}); } + + auto InEdges(storage::v3::View view, const std::vector &edge_types, + const VertexAccessor &dest) const + -> storage::v3::Result { + auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_); + if (maybe_edges.HasError()) return maybe_edges.GetError(); + return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges)); + } + + auto OutEdges(storage::v3::View view, const std::vector &edge_types) const + -> storage::v3::Result { + auto maybe_edges = impl_.OutEdges(view, edge_types); + if (maybe_edges.HasError()) return maybe_edges.GetError(); + return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges)); + } + + auto OutEdges(storage::v3::View view) const { return OutEdges(view, {}); } + + auto OutEdges(storage::v3::View view, const std::vector &edge_types, + const VertexAccessor &dest) const + -> storage::v3::Result { + auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_); + if (maybe_edges.HasError()) return maybe_edges.GetError(); + return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges)); + } + + storage::v3::Result InDegree(storage::v3::View view) const { return impl_.InDegree(view); } + + storage::v3::Result OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); } + + int64_t CypherId() const { return impl_.Gid().AsInt(); } + + storage::v3::Gid Gid() const noexcept { return impl_.Gid(); } + + bool operator==(const VertexAccessor &v) const noexcept { + static_assert(noexcept(impl_ == v.impl_)); + return impl_ == v.impl_; + } + + bool operator!=(const VertexAccessor &v) const noexcept { return !(*this == v); } +}; + +inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); } + +inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); } + +inline bool EdgeAccessor::IsCycle() const { return To() == From(); } + +class DbAccessor final { + storage::v3::Storage::Accessor *accessor_; + + class VerticesIterable final { + storage::v3::VerticesIterable iterable_; + + public: + class Iterator final { + storage::v3::VerticesIterable::Iterator it_; + + public: + explicit Iterator(storage::v3::VerticesIterable::Iterator it) : it_(it) {} + + VertexAccessor operator*() const { return VertexAccessor(*it_); } + + Iterator &operator++() { + ++it_; + return *this; + } + + bool operator==(const Iterator &other) const { return it_ == other.it_; } + + bool operator!=(const Iterator &other) const { return !(other == *this); } + }; + + explicit VerticesIterable(storage::v3::VerticesIterable iterable) : iterable_(std::move(iterable)) {} + + Iterator begin() { return Iterator(iterable_.begin()); } + + Iterator end() { return Iterator(iterable_.end()); } + }; + + public: + explicit DbAccessor(storage::v3::Storage::Accessor *accessor) : accessor_(accessor) {} + + std::optional FindVertex(storage::v3::Gid gid, storage::v3::View view) { + auto maybe_vertex = accessor_->FindVertex(gid, view); + if (maybe_vertex) return VertexAccessor(*maybe_vertex); + return std::nullopt; + } + + void FinalizeTransaction() { accessor_->FinalizeTransaction(); } + + VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); } + + VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) { + return VerticesIterable(accessor_->Vertices(label, view)); + } + + VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property) { + return VerticesIterable(accessor_->Vertices(label, property, view)); + } + + VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property, + const storage::v3::PropertyValue &value) { + return VerticesIterable(accessor_->Vertices(label, property, value, view)); + } + + VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property, + const std::optional> &lower, + const std::optional> &upper) { + return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view)); + } + + VertexAccessor InsertVertex() { return VertexAccessor(accessor_->CreateVertex()); } + + storage::v3::Result InsertEdge(VertexAccessor *from, VertexAccessor *to, + const storage::v3::EdgeTypeId &edge_type) { + auto maybe_edge = accessor_->CreateEdge(&from->impl_, &to->impl_, edge_type); + if (maybe_edge.HasError()) return storage::v3::Result(maybe_edge.GetError()); + return EdgeAccessor(*maybe_edge); + } + + storage::v3::Result> RemoveEdge(EdgeAccessor *edge) { + auto res = accessor_->DeleteEdge(&edge->impl_); + if (res.HasError()) { + return res.GetError(); + } + + const auto &value = res.GetValue(); + if (!value) { + return std::optional{}; + } + + return std::make_optional(*value); + } + + storage::v3::Result>>> DetachRemoveVertex( + VertexAccessor *vertex_accessor) { + using ReturnType = std::pair>; + + auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_); + if (res.HasError()) { + return res.GetError(); + } + + const auto &value = res.GetValue(); + if (!value) { + return std::optional{}; + } + + const auto &[vertex, edges] = *value; + + std::vector deleted_edges; + deleted_edges.reserve(edges.size()); + std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges), + [](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; }); + + return std::make_optional(vertex, std::move(deleted_edges)); + } + + storage::v3::Result> RemoveVertex(VertexAccessor *vertex_accessor) { + auto res = accessor_->DeleteVertex(&vertex_accessor->impl_); + if (res.HasError()) { + return res.GetError(); + } + + const auto &value = res.GetValue(); + if (!value) { + return std::optional{}; + } + + return std::make_optional(*value); + } + + storage::v3::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); } + + storage::v3::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); } + + storage::v3::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); } + + const std::string &PropertyToName(storage::v3::PropertyId prop) const { return accessor_->PropertyToName(prop); } + + const std::string &LabelToName(storage::v3::LabelId label) const { return accessor_->LabelToName(label); } + + const std::string &EdgeTypeToName(storage::v3::EdgeTypeId type) const { return accessor_->EdgeTypeToName(type); } + + void AdvanceCommand() { accessor_->AdvanceCommand(); } + + utils::BasicResult Commit() { return accessor_->Commit(); } + + void Abort() { accessor_->Abort(); } + + bool LabelIndexExists(storage::v3::LabelId label) const { return accessor_->LabelIndexExists(label); } + + bool LabelPropertyIndexExists(storage::v3::LabelId label, storage::v3::PropertyId prop) const { + return accessor_->LabelPropertyIndexExists(label, prop); + } + + int64_t VerticesCount() const { return accessor_->ApproximateVertexCount(); } + + int64_t VerticesCount(storage::v3::LabelId label) const { return accessor_->ApproximateVertexCount(label); } + + int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) const { + return accessor_->ApproximateVertexCount(label, property); + } + + int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property, + const storage::v3::PropertyValue &value) const { + return accessor_->ApproximateVertexCount(label, property, value); + } + + int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property, + const std::optional> &lower, + const std::optional> &upper) const { + return accessor_->ApproximateVertexCount(label, property, lower, upper); + } + + storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); } + + storage::v3::ConstraintsInfo ListAllConstraints() const { return accessor_->ListAllConstraints(); } +}; + +} // namespace memgraph::query::v2 + +namespace std { + +template <> +struct hash { + size_t operator()(const memgraph::query::v2::VertexAccessor &v) const { + return std::hash{}(v.impl_); + } +}; + +template <> +struct hash { + size_t operator()(const memgraph::query::v2::EdgeAccessor &e) const { + return std::hash{}(e.impl_); + } +}; + +} // namespace std diff --git a/src/query/v2/discard_value_stream.hpp b/src/query/v2/discard_value_stream.hpp new file mode 100644 index 000000000..8703aa470 --- /dev/null +++ b/src/query/v2/discard_value_stream.hpp @@ -0,0 +1,24 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "query/v2/typed_value.hpp" + +namespace memgraph::query::v2 { +struct DiscardValueResultStream final { + void Result(const std::vector & /*values*/) { + // do nothing + } +}; +} // namespace memgraph::query::v2 diff --git a/src/query/v2/dump.cpp b/src/query/v2/dump.cpp new file mode 100644 index 000000000..e155c600d --- /dev/null +++ b/src/query/v2/dump.cpp @@ -0,0 +1,541 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/dump.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "query/v2/db_accessor.hpp" +#include "query/v2/exceptions.hpp" +#include "query/v2/stream.hpp" +#include "query/v2/typed_value.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/storage.hpp" +#include "utils/algorithm.hpp" +#include "utils/logging.hpp" +#include "utils/string.hpp" +#include "utils/temporal.hpp" + +namespace memgraph::query::v2 { + +namespace { + +// Property that is used to make a difference among vertices. It is added to +// property set of vertices to match edges and removed after the entire graph +// is built. +const char *kInternalPropertyId = "__mg_id__"; + +// Label that is attached to each vertex and is used for easier creation of +// index on internal property id. +const char *kInternalVertexLabel = "__mg_vertex__"; + +/// A helper function that escapes label, edge type and property names. +std::string EscapeName(const std::string_view value) { + std::string out; + out.reserve(value.size() + 2); + out.append(1, '`'); + for (auto c : value) { + if (c == '`') { + out.append("``"); + } else { + out.append(1, c); + } + } + out.append(1, '`'); + return out; +} + +void DumpPreciseDouble(std::ostream *os, double value) { + // A temporary stream is used to keep precision of the original output + // stream unchanged. + std::ostringstream temp_oss; + temp_oss << std::setprecision(std::numeric_limits::max_digits10) << value; + *os << temp_oss.str(); +} + +namespace { +void DumpDate(std::ostream &os, const storage::v3::TemporalData &value) { + utils::Date date(value.microseconds); + os << "DATE(\"" << date << "\")"; +} + +void DumpLocalTime(std::ostream &os, const storage::v3::TemporalData &value) { + utils::LocalTime lt(value.microseconds); + os << "LOCALTIME(\"" << lt << "\")"; +} + +void DumpLocalDateTime(std::ostream &os, const storage::v3::TemporalData &value) { + utils::LocalDateTime ldt(value.microseconds); + os << "LOCALDATETIME(\"" << ldt << "\")"; +} + +void DumpDuration(std::ostream &os, const storage::v3::TemporalData &value) { + utils::Duration dur(value.microseconds); + os << "DURATION(\"" << dur << "\")"; +} + +void DumpTemporalData(std::ostream &os, const storage::v3::TemporalData &value) { + switch (value.type) { + case storage::v3::TemporalType::Date: { + DumpDate(os, value); + return; + } + case storage::v3::TemporalType::LocalTime: { + DumpLocalTime(os, value); + return; + } + case storage::v3::TemporalType::LocalDateTime: { + DumpLocalDateTime(os, value); + return; + } + case storage::v3::TemporalType::Duration: { + DumpDuration(os, value); + return; + } + } +} +} // namespace + +void DumpPropertyValue(std::ostream *os, const storage::v3::PropertyValue &value) { + switch (value.type()) { + case storage::v3::PropertyValue::Type::Null: + *os << "Null"; + return; + case storage::v3::PropertyValue::Type::Bool: + *os << (value.ValueBool() ? "true" : "false"); + return; + case storage::v3::PropertyValue::Type::String: + *os << utils::Escape(value.ValueString()); + return; + case storage::v3::PropertyValue::Type::Int: + *os << value.ValueInt(); + return; + case storage::v3::PropertyValue::Type::Double: + DumpPreciseDouble(os, value.ValueDouble()); + return; + case storage::v3::PropertyValue::Type::List: { + *os << "["; + const auto &list = value.ValueList(); + utils::PrintIterable(*os, list, ", ", [](auto &os, const auto &item) { DumpPropertyValue(&os, item); }); + *os << "]"; + return; + } + case storage::v3::PropertyValue::Type::Map: { + *os << "{"; + const auto &map = value.ValueMap(); + utils::PrintIterable(*os, map, ", ", [](auto &os, const auto &kv) { + os << EscapeName(kv.first) << ": "; + DumpPropertyValue(&os, kv.second); + }); + *os << "}"; + return; + } + case storage::v3::PropertyValue::Type::TemporalData: { + DumpTemporalData(*os, value.ValueTemporalData()); + return; + } + } +} + +void DumpProperties(std::ostream *os, query::v2::DbAccessor *dba, + const std::map &store, + std::optional property_id = std::nullopt) { + *os << "{"; + if (property_id) { + *os << kInternalPropertyId << ": " << *property_id; + if (store.size() > 0) *os << ", "; + } + utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) { + os << EscapeName(dba->PropertyToName(kv.first)) << ": "; + DumpPropertyValue(&os, kv.second); + }); + *os << "}"; +} + +void DumpVertex(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::VertexAccessor &vertex) { + *os << "CREATE ("; + *os << ":" << kInternalVertexLabel; + auto maybe_labels = vertex.Labels(storage::v3::View::OLD); + if (maybe_labels.HasError()) { + switch (maybe_labels.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get labels from a deleted node."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get labels from a node that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw query::v2::QueryRuntimeException("Unexpected error when getting labels."); + } + } + for (const auto &label : *maybe_labels) { + *os << ":" << EscapeName(dba->LabelToName(label)); + } + *os << " "; + auto maybe_props = vertex.Properties(storage::v3::View::OLD); + if (maybe_props.HasError()) { + switch (maybe_props.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get properties from a node that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw query::v2::QueryRuntimeException("Unexpected error when getting properties."); + } + } + DumpProperties(os, dba, *maybe_props, vertex.CypherId()); + *os << ");"; +} + +void DumpEdge(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::EdgeAccessor &edge) { + *os << "MATCH "; + *os << "(u:" << kInternalVertexLabel << "), "; + *os << "(v:" << kInternalVertexLabel << ")"; + *os << " WHERE "; + *os << "u." << kInternalPropertyId << " = " << edge.From().CypherId(); + *os << " AND "; + *os << "v." << kInternalPropertyId << " = " << edge.To().CypherId() << " "; + *os << "CREATE (u)-["; + *os << ":" << EscapeName(dba->EdgeTypeToName(edge.EdgeType())); + auto maybe_props = edge.Properties(storage::v3::View::OLD); + if (maybe_props.HasError()) { + switch (maybe_props.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get properties from an edge that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw query::v2::QueryRuntimeException("Unexpected error when getting properties."); + } + } + if (maybe_props->size() > 0) { + *os << " "; + DumpProperties(os, dba, *maybe_props); + } + *os << "]->(v);"; +} + +void DumpLabelIndex(std::ostream *os, query::v2::DbAccessor *dba, const storage::v3::LabelId label) { + *os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << ";"; +} + +void DumpLabelPropertyIndex(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label, + storage::v3::PropertyId property) { + *os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << "(" << EscapeName(dba->PropertyToName(property)) + << ");"; +} + +void DumpExistenceConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label, + storage::v3::PropertyId property) { + *os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT EXISTS (u." + << EscapeName(dba->PropertyToName(property)) << ");"; +} + +void DumpUniqueConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label, + const std::set &properties) { + *os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT "; + utils::PrintIterable(*os, properties, ", ", [&dba](auto &stream, const auto &property) { + stream << "u." << EscapeName(dba->PropertyToName(property)); + }); + *os << " IS UNIQUE;"; +} + +} // namespace + +PullPlanDump::PullPlanDump(DbAccessor *dba) + : dba_(dba), + vertices_iterable_(dba->Vertices(storage::v3::View::OLD)), + pull_chunks_{// Dump all label indices + CreateLabelIndicesPullChunk(), + // Dump all label property indices + CreateLabelPropertyIndicesPullChunk(), + // Dump all existence constraints + CreateExistenceConstraintsPullChunk(), + // Dump all unique constraints + CreateUniqueConstraintsPullChunk(), + // Create internal index for faster edge creation + CreateInternalIndexPullChunk(), + // Dump all vertices + CreateVertexPullChunk(), + // Dump all edges + CreateEdgePullChunk(), + // Drop the internal index + CreateDropInternalIndexPullChunk(), + // Internal index cleanup + CreateInternalIndexCleanupPullChunk()} {} + +bool PullPlanDump::Pull(AnyStream *stream, std::optional n) { + // Iterate all functions that stream some results. + // Each function should return number of results it streamed after it + // finishes. If the function did not finish streaming all the results, + // std::nullopt should be returned because n results have already been sent. + while (current_chunk_index_ < pull_chunks_.size() && (!n || *n > 0)) { + const auto maybe_streamed_count = pull_chunks_[current_chunk_index_](stream, n); + + if (!maybe_streamed_count) { + // n wasn't large enough to stream all the results from the current chunk + break; + } + + if (n) { + // chunk finished streaming its results + // subtract number of results streamed in current pull + // so we know how many results we need to stream from future + // chunks. + *n -= *maybe_streamed_count; + } + + ++current_chunk_index_; + } + return current_chunk_index_ == pull_chunks_.size(); +} + +PullPlanDump::PullChunk PullPlanDump::CreateLabelIndicesPullChunk() { + // Dump all label indices + return [this, global_index = 0U](AnyStream *stream, std::optional n) mutable -> std::optional { + // Delay the construction of indices vectors + if (!indices_info_) { + indices_info_.emplace(dba_->ListAllIndices()); + } + const auto &label = indices_info_->label; + + size_t local_counter = 0; + while (global_index < label.size() && (!n || local_counter < *n)) { + std::ostringstream os; + DumpLabelIndex(&os, dba_, label[global_index]); + stream->Result({TypedValue(os.str())}); + + ++global_index; + ++local_counter; + } + + if (global_index == label.size()) { + return local_counter; + } + + return std::nullopt; + }; +} + +PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() { + return [this, global_index = 0U](AnyStream *stream, std::optional n) mutable -> std::optional { + // Delay the construction of indices vectors + if (!indices_info_) { + indices_info_.emplace(dba_->ListAllIndices()); + } + const auto &label_property = indices_info_->label_property; + + size_t local_counter = 0; + while (global_index < label_property.size() && (!n || local_counter < *n)) { + std::ostringstream os; + const auto &label_property_index = label_property[global_index]; + DumpLabelPropertyIndex(&os, dba_, label_property_index.first, label_property_index.second); + stream->Result({TypedValue(os.str())}); + + ++global_index; + ++local_counter; + } + + if (global_index == label_property.size()) { + return local_counter; + } + + return std::nullopt; + }; +} + +PullPlanDump::PullChunk PullPlanDump::CreateExistenceConstraintsPullChunk() { + return [this, global_index = 0U](AnyStream *stream, std::optional n) mutable -> std::optional { + // Delay the construction of constraint vectors + if (!constraints_info_) { + constraints_info_.emplace(dba_->ListAllConstraints()); + } + + const auto &existence = constraints_info_->existence; + size_t local_counter = 0; + while (global_index < existence.size() && (!n || local_counter < *n)) { + const auto &constraint = existence[global_index]; + std::ostringstream os; + DumpExistenceConstraint(&os, dba_, constraint.first, constraint.second); + stream->Result({TypedValue(os.str())}); + + ++global_index; + ++local_counter; + } + + if (global_index == existence.size()) { + return local_counter; + } + + return std::nullopt; + }; +} + +PullPlanDump::PullChunk PullPlanDump::CreateUniqueConstraintsPullChunk() { + return [this, global_index = 0U](AnyStream *stream, std::optional n) mutable -> std::optional { + // Delay the construction of constraint vectors + if (!constraints_info_) { + constraints_info_.emplace(dba_->ListAllConstraints()); + } + + const auto &unique = constraints_info_->unique; + size_t local_counter = 0; + while (global_index < unique.size() && (!n || local_counter < *n)) { + const auto &constraint = unique[global_index]; + std::ostringstream os; + DumpUniqueConstraint(&os, dba_, constraint.first, constraint.second); + stream->Result({TypedValue(os.str())}); + + ++global_index; + ++local_counter; + } + + if (global_index == unique.size()) { + return local_counter; + } + + return std::nullopt; + }; +} + +PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() { + return [this](AnyStream *stream, std::optional) mutable -> std::optional { + if (vertices_iterable_.begin() != vertices_iterable_.end()) { + std::ostringstream os; + os << "CREATE INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");"; + stream->Result({TypedValue(os.str())}); + internal_index_created_ = true; + return 1; + } + return 0; + }; +} + +PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() { + return [this, maybe_current_iter = std::optional{}]( + AnyStream *stream, std::optional n) mutable -> std::optional { + // Delay the call of begin() function + // If multiple begins are called before an iteration, + // one iteration will make the rest of iterators be in undefined + // states. + if (!maybe_current_iter) { + maybe_current_iter.emplace(vertices_iterable_.begin()); + } + + auto ¤t_iter{*maybe_current_iter}; + + size_t local_counter = 0; + while (current_iter != vertices_iterable_.end() && (!n || local_counter < *n)) { + std::ostringstream os; + DumpVertex(&os, dba_, *current_iter); + stream->Result({TypedValue(os.str())}); + ++local_counter; + ++current_iter; + } + if (current_iter == vertices_iterable_.end()) { + return local_counter; + } + + return std::nullopt; + }; +} + +PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() { + return [this, maybe_current_vertex_iter = std::optional{}, + // we need to save the iterable which contains list of accessor so + // our saved iterator is valid in the next run + maybe_edge_iterable = std::shared_ptr{nullptr}, + maybe_current_edge_iter = std::optional{}]( + AnyStream *stream, std::optional n) mutable -> std::optional { + // Delay the call of begin() function + // If multiple begins are called before an iteration, + // one iteration will make the rest of iterators be in undefined + // states. + if (!maybe_current_vertex_iter) { + maybe_current_vertex_iter.emplace(vertices_iterable_.begin()); + } + + auto ¤t_vertex_iter{*maybe_current_vertex_iter}; + size_t local_counter = 0U; + for (; current_vertex_iter != vertices_iterable_.end() && (!n || local_counter < *n); ++current_vertex_iter) { + const auto &vertex = *current_vertex_iter; + // If we have a saved iterable from a previous pull + // we need to use the same iterable + if (!maybe_edge_iterable) { + maybe_edge_iterable = std::make_shared(vertex.OutEdges(storage::v3::View::OLD)); + } + auto &maybe_edges = *maybe_edge_iterable; + MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!"); + auto current_edge_iter = maybe_current_edge_iter ? *maybe_current_edge_iter : maybe_edges->begin(); + for (; current_edge_iter != maybe_edges->end() && (!n || local_counter < *n); ++current_edge_iter) { + std::ostringstream os; + DumpEdge(&os, dba_, *current_edge_iter); + stream->Result({TypedValue(os.str())}); + + ++local_counter; + } + + if (current_edge_iter != maybe_edges->end()) { + maybe_current_edge_iter.emplace(current_edge_iter); + return std::nullopt; + } + + maybe_current_edge_iter = std::nullopt; + maybe_edge_iterable = nullptr; + } + + if (current_vertex_iter == vertices_iterable_.end()) { + return local_counter; + } + + return std::nullopt; + }; +} + +PullPlanDump::PullChunk PullPlanDump::CreateDropInternalIndexPullChunk() { + return [this](AnyStream *stream, std::optional) { + if (internal_index_created_) { + std::ostringstream os; + os << "DROP INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");"; + stream->Result({TypedValue(os.str())}); + return 1; + } + return 0; + }; +} + +PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexCleanupPullChunk() { + return [this](AnyStream *stream, std::optional) { + if (internal_index_created_) { + std::ostringstream os; + os << "MATCH (u) REMOVE u:" << kInternalVertexLabel << ", u." << kInternalPropertyId << ";"; + stream->Result({TypedValue(os.str())}); + return 1; + } + return 0; + }; +} + +void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream) { PullPlanDump(dba).Pull(stream, {}); } + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/dump.hpp b/src/query/v2/dump.hpp new file mode 100644 index 000000000..de8018724 --- /dev/null +++ b/src/query/v2/dump.hpp @@ -0,0 +1,66 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "query/v2/db_accessor.hpp" +#include "query/v2/stream.hpp" +#include "storage/v3/storage.hpp" + +namespace memgraph::query::v2 { + +void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream); + +struct PullPlanDump { + explicit PullPlanDump(query::v2::DbAccessor *dba); + + /// Pull the dump results lazily + /// @return true if all results were returned, false otherwise + bool Pull(AnyStream *stream, std::optional n); + + private: + query::v2::DbAccessor *dba_ = nullptr; + + std::optional indices_info_ = std::nullopt; + std::optional constraints_info_ = std::nullopt; + + using VertexAccessorIterable = decltype(std::declval().Vertices(storage::v3::View::OLD)); + using VertexAccessorIterableIterator = decltype(std::declval().begin()); + + using EdgeAccessorIterable = decltype(std::declval().OutEdges(storage::v3::View::OLD)); + using EdgeAccessorIterableIterator = decltype(std::declval().GetValue().begin()); + + VertexAccessorIterable vertices_iterable_; + bool internal_index_created_ = false; + + size_t current_chunk_index_ = 0; + + using PullChunk = std::function(AnyStream *stream, std::optional n)>; + // We define every part of the dump query in a self contained function. + // Each functions is responsible of keeping track of its execution status. + // If a function did finish its execution, it should return number of results + // it streamed so we know how many rows should be pulled from the next + // function, otherwise std::nullopt is returned. + std::vector pull_chunks_; + + PullChunk CreateLabelIndicesPullChunk(); + PullChunk CreateLabelPropertyIndicesPullChunk(); + PullChunk CreateExistenceConstraintsPullChunk(); + PullChunk CreateUniqueConstraintsPullChunk(); + PullChunk CreateInternalIndexPullChunk(); + PullChunk CreateVertexPullChunk(); + PullChunk CreateEdgePullChunk(); + PullChunk CreateDropInternalIndexPullChunk(); + PullChunk CreateInternalIndexCleanupPullChunk(); +}; +} // namespace memgraph::query::v2 diff --git a/src/query/v2/exceptions.hpp b/src/query/v2/exceptions.hpp new file mode 100644 index 000000000..e0802a6cc --- /dev/null +++ b/src/query/v2/exceptions.hpp @@ -0,0 +1,227 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "utils/exceptions.hpp" + +#include + +namespace memgraph::query::v2 { + +/** + * @brief Base class of all query language related exceptions. All exceptions + * derived from this one will be interpreted as ClientError-s, i. e. if client + * executes same query again without making modifications to the database data, + * query will fail again. + */ +class QueryException : public utils::BasicException { + using utils::BasicException::BasicException; +}; + +class LexingException : public QueryException { + public: + using QueryException::QueryException; + LexingException() : QueryException("") {} +}; + +class SyntaxException : public QueryException { + public: + using QueryException::QueryException; + SyntaxException() : QueryException("") {} +}; + +// TODO: Figure out what information to put in exception. +// Error reporting is tricky since we get stripped query and position of error +// in original query is not same as position of error in stripped query. Most +// correct approach would be to do semantic analysis with original query even +// for already hashed queries, but that has obvious performance issues. Other +// approach would be to report some of the semantic errors in runtime of the +// query and only report line numbers of semantic errors (not position in the +// line) if multiple line strings are not allowed by grammar. We could also +// print whole line that contains error instead of specifying line number. +class SemanticException : public QueryException { + public: + using QueryException::QueryException; + SemanticException() : QueryException("") {} +}; + +class UnboundVariableError : public SemanticException { + public: + explicit UnboundVariableError(const std::string &name) : SemanticException("Unbound variable: " + name + ".") {} +}; + +class RedeclareVariableError : public SemanticException { + public: + explicit RedeclareVariableError(const std::string &name) : SemanticException("Redeclaring variable: " + name + ".") {} +}; + +class TypeMismatchError : public SemanticException { + public: + TypeMismatchError(const std::string &name, const std::string &datum, const std::string &expected) + : SemanticException(fmt::format("Type mismatch: {} already defined as {}, expected {}.", name, datum, expected)) { + } +}; + +class UnprovidedParameterError : public QueryException { + public: + using QueryException::QueryException; +}; + +class ProfileInMulticommandTxException : public QueryException { + public: + using QueryException::QueryException; + ProfileInMulticommandTxException() : QueryException("PROFILE not allowed in multicommand transactions.") {} +}; + +class IndexInMulticommandTxException : public QueryException { + public: + using QueryException::QueryException; + IndexInMulticommandTxException() : QueryException("Index manipulation not allowed in multicommand transactions.") {} +}; + +class ConstraintInMulticommandTxException : public QueryException { + public: + using QueryException::QueryException; + ConstraintInMulticommandTxException() + : QueryException( + "Constraint manipulation not allowed in multicommand " + "transactions.") {} +}; + +class InfoInMulticommandTxException : public QueryException { + public: + using QueryException::QueryException; + InfoInMulticommandTxException() : QueryException("Info reporting not allowed in multicommand transactions.") {} +}; + +/** + * An exception for an illegal operation that can not be detected + * before the query starts executing over data. + */ +class QueryRuntimeException : public QueryException { + public: + using QueryException::QueryException; +}; + +// This one is inherited from BasicException and will be treated as +// TransientError, i. e. client will be encouraged to retry execution because it +// could succeed if executed again. +class HintedAbortError : public utils::BasicException { + public: + using utils::BasicException::BasicException; + HintedAbortError() + : utils::BasicException( + "Transaction was asked to abort, most likely because it was " + "executing longer than time specified by " + "--query-execution-timeout-sec flag.") {} +}; + +class ExplicitTransactionUsageException : public QueryRuntimeException { + public: + using QueryRuntimeException::QueryRuntimeException; +}; + +/** + * An exception for serialization error + */ +class TransactionSerializationException : public QueryException { + public: + using QueryException::QueryException; + TransactionSerializationException() + : QueryException( + "Cannot resolve conflicting transactions. You can retry this transaction when the conflicting transaction " + "is finished") {} +}; + +class ReconstructionException : public QueryException { + public: + ReconstructionException() + : QueryException( + "Record invalid after WITH clause. Most likely deleted by a " + "preceeding DELETE.") {} +}; + +class RemoveAttachedVertexException : public QueryRuntimeException { + public: + RemoveAttachedVertexException() + : QueryRuntimeException( + "Failed to remove node because of it's existing " + "connections. Consider using DETACH DELETE.") {} +}; + +class UserModificationInMulticommandTxException : public QueryException { + public: + UserModificationInMulticommandTxException() + : QueryException("Authentication clause not allowed in multicommand transactions.") {} +}; + +class InvalidArgumentsException : public QueryException { + public: + InvalidArgumentsException(const std::string &argument_name, const std::string &message) + : QueryException(fmt::format("Invalid arguments sent: {} - {}", argument_name, message)) {} +}; + +class ReplicationModificationInMulticommandTxException : public QueryException { + public: + ReplicationModificationInMulticommandTxException() + : QueryException("Replication clause not allowed in multicommand transactions.") {} +}; + +class LockPathModificationInMulticommandTxException : public QueryException { + public: + LockPathModificationInMulticommandTxException() + : QueryException("Lock path query not allowed in multicommand transactions.") {} +}; + +class FreeMemoryModificationInMulticommandTxException : public QueryException { + public: + FreeMemoryModificationInMulticommandTxException() + : QueryException("Free memory query not allowed in multicommand transactions.") {} +}; + +class TriggerModificationInMulticommandTxException : public QueryException { + public: + TriggerModificationInMulticommandTxException() + : QueryException("Trigger queries not allowed in multicommand transactions.") {} +}; + +class StreamQueryInMulticommandTxException : public QueryException { + public: + StreamQueryInMulticommandTxException() + : QueryException("Stream queries are not allowed in multicommand transactions.") {} +}; + +class IsolationLevelModificationInMulticommandTxException : public QueryException { + public: + IsolationLevelModificationInMulticommandTxException() + : QueryException("Isolation level cannot be modified in multicommand transactions.") {} +}; + +class CreateSnapshotInMulticommandTxException final : public QueryException { + public: + CreateSnapshotInMulticommandTxException() + : QueryException("Snapshot cannot be created in multicommand transactions.") {} +}; + +class SettingConfigInMulticommandTxException final : public QueryException { + public: + SettingConfigInMulticommandTxException() + : QueryException("Settings cannot be changed or fetched in multicommand transactions.") {} +}; + +class VersionInfoInMulticommandTxException : public QueryException { + public: + VersionInfoInMulticommandTxException() + : QueryException("Version info query not allowed in multicommand transactions.") {} +}; + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/ast/ast.lcp b/src/query/v2/frontend/ast/ast.lcp new file mode 100644 index 000000000..b858ab71f --- /dev/null +++ b/src/query/v2/frontend/ast/ast.lcp @@ -0,0 +1,2676 @@ +;; Copyright 2022 Memgraph Ltd. +;; +;; Use of this software is governed by the Business Source License +;; included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +;; License, and you may not use this file except in compliance with the Business Source License. +;; +;; As of the Change Date specified in that file, in accordance with +;; the Business Source License, use of this software will be governed +;; by the Apache License, Version 2.0, included in the file +;; licenses/APL.txt. + +#>cpp +#pragma once + +#include +#include +#include +#include + +#include "query/v2/frontend/ast/ast_visitor.hpp" +#include "query/v2/frontend/semantic/symbol.hpp" +#include "query/v2/interpret/awesome_memgraph_functions.hpp" +#include "query/v2/typed_value.hpp" +#include "storage/v3/property_value.hpp" +#include "utils/typeinfo.hpp" + +cpp<# + +(lcp:namespace memgraph) +(lcp:namespace query) +(lcp:namespace v2) + +(defun slk-save-ast-pointer (member) + #>cpp + query::v2::SaveAstPointer(self.${member}, builder); + cpp<#) + +(defun slk-load-ast-pointer (type) + (lambda (member) + #>cpp + self->${member} = query::v2::LoadAstPointer(storage, reader); + cpp<#)) + +(defun slk-save-ast-vector (member) + #>cpp + size_t size = self.${member}.size(); + slk::Save(size, builder); + for (const auto *val : self.${member}) { + query::v2::SaveAstPointer(val, builder); + } + cpp<#) + +(defun slk-load-ast-vector (type) + (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + self->${member}[i] = query::v2::LoadAstPointer(storage, reader); + } + cpp<#)) + +(defun slk-save-property-map (member) + #>cpp + size_t size = self.${member}.size(); + slk::Save(size, builder); + for (const auto &entry : self.${member}) { + slk::Save(entry.first, builder); + query::v2::SaveAstPointer(entry.second, builder); + } + cpp<#) + +(defun slk-load-property-map (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + for (size_t i = 0; i < size; ++i) { + query::v2::PropertyIx key; + slk::Load(&key, reader, storage); + auto *value = query::v2::LoadAstPointer(storage, reader); + self->${member}.emplace(key, value); + } + cpp<#) + +(defun clone-property-map (source dest) + #>cpp + for (const auto &entry : ${source}) { + PropertyIx key = storage->GetPropertyIx(entry.first.name); + ${dest}[key] = entry.second->Clone(storage); + } + cpp<#) + +(defun slk-save-expression-map (member) + #>cpp + size_t size = self.${member}.size(); + slk::Save(size, builder); + for (const auto &entry : self.${member}) { + query::v2::SaveAstPointer(entry.first, builder); + query::v2::SaveAstPointer(entry.second, builder); + } + cpp<#) + +(defun slk-load-expression-map (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + for (size_t i = 0; i < size; ++i) { + auto *key = query::v2::LoadAstPointer(storage, reader); + auto *value = query::v2::LoadAstPointer(storage, reader); + self->${member}.emplace(key, value); + } + cpp<#) + +(defun clone-expression-map (source dest) + #>cpp + for (const auto &[key, value] : ${source}) { + ${dest}[key->Clone(storage)] = value->Clone(storage); + } + cpp<#) + +(defun slk-load-name-ix (name-type) + (lambda (member) + #>cpp + self->${member} = storage->Get${name-type}Ix(self->name).ix; + cpp<#)) + +(defun clone-name-ix-vector (name-type) + (lambda (source dest) + #>cpp + ${dest}.resize(${source}.size()); + for (auto i = 0; i < ${dest}.size(); ++i) { + ${dest}[i] = storage->Get${name-type}Ix(${source}[i].name); + } + cpp<#)) + +;; The following index structs serve as a decoupling point of AST from +;; concrete database types. All the names are collected in AstStorage, and can +;; be indexed through these instances. This means that we can create a vector +;; of concrete database types in the same order as all of the names and use the +;; same index to get the correct behaviour. Additionally, each index is +;; accompanied with the duplicated name found at the same index. The primary +;; reason for this duplication is simplifying the Clone and serialization API. +;; When an old index is being cloned or deserialized into a new AstStorage, we +;; request the new `ix` from the new AstStorage for the same `name`. If we +;; didn't do this, we would have to duplicate the old storage, which would +;; require having access to that storage. This in turn would complicate the +;; client code. +(lcp:define-struct label-ix () + ((name "std::string") + (ix :int64_t + :dont-save t + :slk-load (slk-load-name-ix "Label"))) + (:serialize (:slk :load-args '((storage "query::v2::AstStorage *"))))) + +(lcp:define-struct property-ix () + ((name "std::string") + (ix :int64_t + :dont-save t + :slk-load (slk-load-name-ix "Property"))) + (:serialize (:slk :load-args '((storage "query::v2::AstStorage *"))))) + +(lcp:define-struct edge-type-ix () + ((name "std::string") + (ix :int64_t + :dont-save t + :slk-load (slk-load-name-ix "EdgeType"))) + (:serialize (:slk :load-args '((storage "query::v2::AstStorage *"))))) + +#>cpp +inline bool operator==(const LabelIx &a, const LabelIx &b) { + return a.ix == b.ix && a.name == b.name; +} + +inline bool operator!=(const LabelIx &a, const LabelIx &b) { return !(a == b); } + +inline bool operator==(const PropertyIx &a, const PropertyIx &b) { + return a.ix == b.ix && a.name == b.name; +} + +inline bool operator!=(const PropertyIx &a, const PropertyIx &b) { + return !(a == b); +} + +inline bool operator==(const EdgeTypeIx &a, const EdgeTypeIx &b) { + return a.ix == b.ix && a.name == b.name; +} + +inline bool operator!=(const EdgeTypeIx &a, const EdgeTypeIx &b) { + return !(a == b); +} +cpp<# + +(lcp:pop-namespace) ;; namespace v2 +(lcp:pop-namespace) ;; namespace query +(lcp:pop-namespace) ;; namespace memgraph + +#>cpp +namespace std { + +template <> +struct hash { + size_t operator()(const memgraph::query::v2::LabelIx &label) const { return label.ix; } +}; + +template <> +struct hash { + size_t operator()(const memgraph::query::v2::PropertyIx &prop) const { return prop.ix; } +}; + +template <> +struct hash { + size_t operator()(const memgraph::query::v2::EdgeTypeIx &edge_type) const { + return edge_type.ix; + } +}; + +} // namespace std +cpp<# + +(lcp:namespace memgraph) +(lcp:namespace query) +(lcp:namespace v2) + +#>cpp + +class Tree; + +// It would be better to call this AstTree, but we already have a class Tree, +// which could be renamed to Node or AstTreeNode, but we also have a class +// called NodeAtom... +class AstStorage { + public: + AstStorage() = default; + AstStorage(const AstStorage &) = delete; + AstStorage &operator=(const AstStorage &) = delete; + AstStorage(AstStorage &&) = default; + AstStorage &operator=(AstStorage &&) = default; + + template + T *Create(Args &&... args) { + T *ptr = new T(std::forward(args)...); + std::unique_ptr tmp(ptr); + storage_.emplace_back(std::move(tmp)); + return ptr; + } + + LabelIx GetLabelIx(const std::string &name) { + return LabelIx{name, FindOrAddName(name, &labels_)}; + } + + PropertyIx GetPropertyIx(const std::string &name) { + return PropertyIx{name, FindOrAddName(name, &properties_)}; + } + + EdgeTypeIx GetEdgeTypeIx(const std::string &name) { + return EdgeTypeIx{name, FindOrAddName(name, &edge_types_)}; + } + + std::vector labels_; + std::vector edge_types_; + std::vector properties_; + + // Public only for serialization access + std::vector> storage_; + + private: + int64_t FindOrAddName(const std::string &name, + std::vector *names) { + for (int64_t i = 0; i < names->size(); ++i) { + if ((*names)[i] == name) { + return i; + } + } + names->push_back(name); + return names->size() - 1; + } +}; +cpp<# + +(lcp:define-class tree () + () + (:abstractp t) + (:public + #>cpp + Tree() = default; + virtual ~Tree() {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :load-args '((storage "query::v2::AstStorage *")))) + (:clone :return-type (lambda (typename) + (format nil "~A*" typename)) + :args '((storage "AstStorage *")) + :init-object (lambda (var typename) + (format nil "~A* ~A = storage->Create<~A>();" + typename var typename)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Expressions +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(lcp:define-class expression (tree "::utils::Visitable" + "::utils::Visitable>" + "::utils::Visitable>") + () + (:abstractp t) + (:public + #>cpp + using utils::Visitable::Accept; + using utils::Visitable>::Accept; + using utils::Visitable>::Accept; + + Expression() = default; + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +(lcp:define-class where (tree "::utils::Visitable") + ((expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:public + #>cpp + using utils::Visitable::Accept; + + Where() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit Where(Expression *expression) : expression_(expression) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +(lcp:define-class binary-operator (expression) + ((expression1 "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (expression2 "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:abstractp t) + (:public + #>cpp + BinaryOperator() = default; + cpp<#) + (:protected + #>cpp + BinaryOperator(Expression *expression1, Expression *expression2) + : expression1_(expression1), expression2_(expression2) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class unary-operator (expression) + ((expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:abstractp t) + (:public + #>cpp + UnaryOperator() = default; + cpp<#) + (:protected + #>cpp + explicit UnaryOperator(Expression *expression) : expression_(expression) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(macrolet ((define-binary-operators () + `(lcp:cpp-list + ,@(loop for op in + '(or-operator xor-operator and-operator addition-operator + subtraction-operator multiplication-operator division-operator + mod-operator not-equal-operator equal-operator less-operator + greater-operator less-equal-operator greater-equal-operator + in-list-operator subscript-operator) + collecting + `(lcp:define-class ,op (binary-operator) + () + (:public + #>cpp + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + expression1_->Accept(visitor) && expression2_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + using BinaryOperator::BinaryOperator; + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)))))) + (define-binary-operators)) + +(macrolet ((define-unary-operators () + `(lcp:cpp-list + ,@(loop for op in + '(not-operator unary-plus-operator + unary-minus-operator is-null-operator) + collecting + `(lcp:define-class ,op (unary-operator) + () + (:public + #>cpp + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + using UnaryOperator::UnaryOperator; + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)))))) + (define-unary-operators)) + +(lcp:define-class aggregation (binary-operator) + ((op "Op" :scope :public) + (symbol-pos :int32_t :initval -1 :scope :public + :documentation "Symbol table position of the symbol this Aggregation is mapped to.")) + (:public + (lcp:define-enum op + (count min max sum avg collect-list collect-map) + (:serialize)) + #>cpp + Aggregation() = default; + + static const constexpr char *const kCount = "COUNT"; + static const constexpr char *const kMin = "MIN"; + static const constexpr char *const kMax = "MAX"; + static const constexpr char *const kSum = "SUM"; + static const constexpr char *const kAvg = "AVG"; + static const constexpr char *const kCollect = "COLLECT"; + + static std::string OpToString(Op op) { + const char *op_strings[] = {kCount, kMin, kMax, kSum, + kAvg, kCollect, kCollect}; + return op_strings[static_cast(op)]; + } + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + if (expression1_) expression1_->Accept(visitor); + if (expression2_) expression2_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + + Aggregation *MapTo(const Symbol &symbol) { + symbol_pos_ = symbol.position(); + return this; + } + cpp<#) + (:protected + #>cpp + // Use only for serialization. + explicit Aggregation(Op op) : op_(op) {} + + /// Aggregation's first expression is the value being aggregated. The second + /// expression is the key used only in COLLECT_MAP. + Aggregation(Expression *expression1, Expression *expression2, Op op) + : BinaryOperator(expression1, expression2), op_(op) { + // COUNT without expression denotes COUNT(*) in cypher. + DMG_ASSERT(expression1 || op == Aggregation::Op::COUNT, + "All aggregations, except COUNT require expression"); + DMG_ASSERT((expression2 == nullptr) ^ (op == Aggregation::Op::COLLECT_MAP), + "The second expression is obligatory in COLLECT_MAP and " + "invalid otherwise"); + } + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class list-slicing-operator (expression) + ((list "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (lower-bound "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (upper-bound "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:public + #>cpp + ListSlicingOperator() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + bool cont = list_->Accept(visitor); + if (cont && lower_bound_) { + cont = lower_bound_->Accept(visitor); + } + if (cont && upper_bound_) { + upper_bound_->Accept(visitor); + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + ListSlicingOperator(Expression *list, Expression *lower_bound, + Expression *upper_bound) + : list_(list), lower_bound_(lower_bound), upper_bound_(upper_bound) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class if-operator (expression) + ((condition "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "None of the expressions should be nullptr. If there is no else_expression, you should make it null PrimitiveLiteral.") + (then-expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (else-expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:public + #>cpp + IfOperator() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + condition_->Accept(visitor) && then_expression_->Accept(visitor) && + else_expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + IfOperator(Expression *condition, Expression *then_expression, + Expression *else_expression) + : condition_(condition), + then_expression_(then_expression), + else_expression_(else_expression) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class base-literal (expression) + () + (:abstractp t) + (:public + #>cpp + BaseLiteral() = default; + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class primitive-literal (base-literal) + ((value "::storage::v3::PropertyValue" :scope :public) + (token-position :int32_t :scope :public :initval -1 + :documentation "This field contains token position of literal used to create PrimitiveLiteral object. If PrimitiveLiteral object is not created from query, leave its value at -1.")) + (:public + #>cpp + PrimitiveLiteral() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(HierarchicalTreeVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:protected + #>cpp + template + explicit PrimitiveLiteral(T value) : value_(value) {} + template + PrimitiveLiteral(T value, int token_position) + : value_(value), token_position_(token_position) {} + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class list-literal (base-literal) + ((elements "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression"))) + (:public + #>cpp + ListLiteral() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + for (auto expr_ptr : elements_) + if (!expr_ptr->Accept(visitor)) break; + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit ListLiteral(const std::vector &elements) + : elements_(elements) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class map-literal (base-literal) + ((elements "std::unordered_map" + :slk-save #'slk-save-property-map + :slk-load #'slk-load-property-map + :clone #'clone-property-map + :scope :public)) + (:public + #>cpp + MapLiteral() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + for (auto pair : elements_) + if (!pair.second->Accept(visitor)) break; + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit MapLiteral( + const std::unordered_map &elements) + : elements_(elements) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class identifier (expression) + ((name "std::string" :scope :public) + (user-declared :bool :initval "true" :scope :public) + (symbol-pos :int32_t :initval -1 :scope :public + :documentation "Symbol table position of the symbol this Identifier is mapped to.")) + (:public + #>cpp + Identifier() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(HierarchicalTreeVisitor); + + Identifier *MapTo(const Symbol &symbol) { + symbol_pos_ = symbol.position(); + return this; + } + + explicit Identifier(const std::string &name) : name_(name) {} + Identifier(const std::string &name, bool user_declared) + : name_(name), user_declared_(user_declared) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class property-lookup (expression) + ((expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (property "PropertyIx" :scope :public + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, storage); + cpp<#) + :clone (lambda (source dest) + #>cpp + ${dest} = storage->GetPropertyIx(${source}.name); + cpp<#))) + (:public + #>cpp + PropertyLookup() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + PropertyLookup(Expression *expression, PropertyIx property) + : expression_(expression), property_(property) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class labels-test (expression) + ((expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (labels "std::vector" :scope :public + :slk-load (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + slk::Load(&self->${member}[i], reader, storage); + } + cpp<#) + :clone (clone-name-ix-vector "Label"))) + (:public + #>cpp + LabelsTest() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + LabelsTest(Expression *expression, const std::vector &labels) + : expression_(expression), labels_(labels) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class function (expression) + ((arguments "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression")) + (function-name "std::string" :scope :public) + (function "std::function" + :scope :public + :dont-save t + :clone :copy + :slk-load (lambda (member) + #>cpp + self->${member} = query::v2::NameToFunction(self->function_name_); + cpp<#))) + (:public + #>cpp + Function() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + for (auto *argument : arguments_) { + if (!argument->Accept(visitor)) break; + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + Function(const std::string &function_name, + const std::vector &arguments) + : arguments_(arguments), + function_name_(function_name), + function_(NameToFunction(function_name_)) { + if (!function_) { + throw SemanticException("Function '{}' doesn't exist.", function_name); + } + } + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class reduce (expression) + ((accumulator "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier") + :documentation "Identifier for the accumulating variable") + (initializer "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Expression which produces the initial accumulator value.") + (identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier") + :documentation "Identifier for the list element.") + (list "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Expression which produces a list to be reduced.") + (expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Expression which does the reduction, i.e. produces the new accumulator value.")) + (:public + #>cpp + Reduce() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + accumulator_->Accept(visitor) && initializer_->Accept(visitor) && + identifier_->Accept(visitor) && list_->Accept(visitor) && + expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + Reduce(Identifier *accumulator, Expression *initializer, Identifier *identifier, + Expression *list, Expression *expression) + : accumulator_(accumulator), + initializer_(initializer), + identifier_(identifier), + list_(list), + expression_(expression) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class coalesce (expression) + ((expressions "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression") + :documentation "A list of expressions to evaluate. None of the expressions should be nullptr.")) + (:public + #>cpp + Coalesce() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + for (auto *expr : expressions_) { + if (!expr->Accept(visitor)) break; + } + } + return visitor.PostVisit(*this); + } + cpp<# + ) + (:private + #>cpp + explicit Coalesce(const std::vector &expressions) + : expressions_(expressions) {} + + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class extract (expression) + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier") + :documentation "Identifier for the list element.") + (list "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Expression which produces a list which will be extracted.") + (expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Expression which produces the new value for list element.")) + (:public + #>cpp + Extract() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + identifier_->Accept(visitor) && list_->Accept(visitor) && + expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + Extract(Identifier *identifier, Expression *list, Expression *expression) + : identifier_(identifier), list_(list), expression_(expression) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class all (expression) + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier")) + (list-expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (where "Where *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Where"))) + (:public + #>cpp + All() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + identifier_->Accept(visitor) && list_expression_->Accept(visitor) && + where_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + All(Identifier *identifier, Expression *list_expression, Where *where) + : identifier_(identifier), + list_expression_(list_expression), + where_(where) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +;; TODO: This is pretty much copy pasted from All. Consider merging Reduce, +;; All, Any, None and Single into something like a higher-order function call +;; which takes a list argument and a function which is applied on list elements. +(lcp:define-class single (expression) + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier")) + (list-expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (where "Where *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Where"))) + (:public + #>cpp + Single() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + identifier_->Accept(visitor) && list_expression_->Accept(visitor) && + where_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + Single(Identifier *identifier, Expression *list_expression, Where *where) + : identifier_(identifier), + list_expression_(list_expression), + where_(where) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +;; TODO: This is pretty much copy pasted from All. Consider merging Reduce, +;; All, Any, None and Single into something like a higher-order function call +;; which takes a list argument and a function which is applied on list elements. +(lcp:define-class any (expression) + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier")) + (list-expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (where "Where *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Where"))) + (:public + #>cpp + Any() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + identifier_->Accept(visitor) && list_expression_->Accept(visitor) && + where_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + Any(Identifier *identifier, Expression *list_expression, Where *where) + : identifier_(identifier), + list_expression_(list_expression), + where_(where) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +;; TODO: This is pretty much copy pasted from All. Consider merging Reduce, +;; All, Any, None and Single into something like a higher-order function call +;; which takes a list argument and a function which is applied on list elements. +(lcp:define-class none (expression) + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier")) + (list-expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (where "Where *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Where"))) + (:public + #>cpp + None() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + identifier_->Accept(visitor) && list_expression_->Accept(visitor) && + where_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + None(Identifier *identifier, Expression *list_expression, Where *where) + : identifier_(identifier), + list_expression_(list_expression), + where_(where) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class parameter-lookup (expression) + ((token-position :int32_t :initval -1 :scope :public + :documentation "This field contains token position of *literal* used to create ParameterLookup object. If ParameterLookup object is not created from a literal leave this value at -1.")) + (:public + #>cpp + ParameterLookup() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(HierarchicalTreeVisitor); + cpp<#) + (:protected + #>cpp + explicit ParameterLookup(int token_position) + : token_position_(token_position) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class regex-match (expression) + ((string-expr "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (regex "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:public + #>cpp + RegexMatch() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + string_expr_->Accept(visitor) && regex_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:private + #>cpp + friend class AstStorage; + RegexMatch(Expression *string_expr, Expression *regex) + : string_expr_(string_expr), regex_(regex) {} + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class named-expression (tree "::utils::Visitable" + "::utils::Visitable>" + "::utils::Visitable>") + ((name "std::string" :scope :public) + (expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (token-position :int32_t :initval -1 :scope :public + :documentation "This field contains token position of first token in named expression used to create name_. If NamedExpression object is not created from query or it is aliased leave this value at -1.") + (symbol-pos :int32_t :initval -1 :scope :public + :documentation "Symbol table position of the symbol this NamedExpression is mapped to.")) + (:public + #>cpp + using utils::Visitable>::Accept; + using utils::Visitable>::Accept; + using utils::Visitable::Accept; + + NamedExpression() = default; + + DEFVISITABLE(ExpressionVisitor); + DEFVISITABLE(ExpressionVisitor); + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + + NamedExpression *MapTo(const Symbol &symbol) { + symbol_pos_ = symbol.position(); + return this; + } + cpp<#) + (:protected + #>cpp + explicit NamedExpression(const std::string &name) : name_(name) {} + NamedExpression(const std::string &name, Expression *expression) + : name_(name), expression_(expression) {} + NamedExpression(const std::string &name, Expression *expression, + int token_position) + : name_(name), expression_(expression), token_position_(token_position) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; END Expressions +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(lcp:define-class pattern-atom (tree "::utils::Visitable") + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier"))) + (:abstractp t) + (:public + #>cpp + using utils::Visitable::Accept; + + PatternAtom() = default; + cpp<#) + (:protected + #>cpp + explicit PatternAtom(Identifier *identifier) : identifier_(identifier) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +(defun clone-variant-properties (source destination) + #>cpp + if (const auto *properties = std::get_if>(&${source})) { + auto &new_obj_properties = std::get>(${destination}); + for (const auto &[property, value_expression] : *properties) { + PropertyIx key = storage->GetPropertyIx(property.name); + new_obj_properties[key] = value_expression->Clone(storage); + } + } else { + ${destination} = std::get(${source})->Clone(storage); + } + cpp<#) + +(lcp:define-class node-atom (pattern-atom) + ((labels "std::vector" :scope :public + :slk-load (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + slk::Load(&self->${member}[i], reader, storage); + } + cpp<#) + :clone (clone-name-ix-vector "Label")) + (properties "std::variant, ParameterLookup*>" + :clone #'clone-variant-properties + :scope :public)) + (:public + #>cpp + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + if (auto* properties = std::get_if>(&properties_)) { + bool cont = identifier_->Accept(visitor); + for (auto &property : *properties) { + if (cont) { + cont = property.second->Accept(visitor); + } + } + } else { + std::get(properties_)->Accept(visitor); + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + using PatternAtom::PatternAtom; + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class edge-atom (pattern-atom) + ((type "Type" :initval "Type::SINGLE" :scope :public) + (direction "Direction" :initval "Direction::BOTH" :scope :public) + (edge-types "std::vector" :scope :public + :slk-load (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + slk::Load(&self->${member}[i], reader, storage); + } + cpp<#) + :clone (clone-name-ix-vector "EdgeType")) + (properties "std::variant, ParameterLookup*>" + :scope :public + :slk-save #'slk-save-property-map + :slk-load #'slk-load-property-map + :clone #'clone-variant-properties) + (lower-bound "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Evaluates to lower bound in variable length expands.") + (upper-bound "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Evaluated to upper bound in variable length expands.") + (filter-lambda "Lambda" :scope :public + :documentation "Filter lambda for variable length expands. Can have an empty expression, but identifiers must be valid, because an optimization pass may inline other expressions into this lambda." + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, storage); + cpp<#)) + (weight-lambda "Lambda" :scope :public + :documentation "Used in weighted shortest path. It must have valid expressions and identifiers. In all other expand types, it is empty." + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, storage); + cpp<#)) + (total-weight "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier") + :documentation "Variable where the total weight for weighted shortest path will be stored.")) + (:public + (lcp:define-enum type + (single depth-first breadth-first weighted-shortest-path) + (:serialize)) + (lcp:define-enum direction + (in out both) + (:serialize)) + (lcp:define-struct lambda () + ((inner-edge "Identifier *" :initval "nullptr" + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier") + :documentation "Argument identifier for the edge currently being traversed.") + (inner-node "Identifier *" :initval "nullptr" + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier") + :documentation "Argument identifier for the destination node of the edge.") + (expression "Expression *" :initval "nullptr" + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Evaluates the result of the lambda.")) + (:documentation "Lambda for use in filtering or weight calculation during variable expand.") + (:serialize (:slk :load-args '((storage "query::v2::AstStorage *")))) + (:clone :args '((storage "AstStorage *")))) + #>cpp + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + bool cont = identifier_->Accept(visitor); + if (auto *properties = std::get_if>(&properties_)) { + for (auto &property : *properties) { + if (cont) { + cont = property.second->Accept(visitor); + } + } + } else { + std::get(properties_)->Accept(visitor); + } + if (cont && lower_bound_) { + cont = lower_bound_->Accept(visitor); + } + if (cont && upper_bound_) { + cont = upper_bound_->Accept(visitor); + } + if (cont && total_weight_) { + total_weight_->Accept(visitor); + } + } + return visitor.PostVisit(*this); + } + + bool IsVariable() const { + switch (type_) { + case Type::DEPTH_FIRST: + case Type::BREADTH_FIRST: + case Type::WEIGHTED_SHORTEST_PATH: + return true; + case Type::SINGLE: + return false; + } + } + cpp<#) + (:protected + #>cpp + using PatternAtom::PatternAtom; + EdgeAtom(Identifier *identifier, Type type, Direction direction) + : PatternAtom(identifier), type_(type), direction_(direction) {} + + // Creates an edge atom for a SINGLE expansion with the given . + EdgeAtom(Identifier *identifier, Type type, Direction direction, + const std::vector &edge_types) + : PatternAtom(identifier), + type_(type), + direction_(direction), + edge_types_(edge_types) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class pattern (tree "::utils::Visitable") + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier")) + (atoms "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "PatternAtom"))) + (:public + #>cpp + using utils::Visitable::Accept; + + Pattern() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + bool cont = identifier_->Accept(visitor); + for (auto &part : atoms_) { + if (cont) { + cont = part->Accept(visitor); + } + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +(lcp:define-class clause (tree "::utils::Visitable") + () + (:abstractp t) + (:public + #>cpp + using utils::Visitable::Accept; + + Clause() = default; + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +(lcp:define-class single-query (tree "::utils::Visitable") + ((clauses "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Clause"))) + (:public + #>cpp + using utils::Visitable::Accept; + + SingleQuery() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + for (auto &clause : clauses_) { + if (!clause->Accept(visitor)) break; + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +(lcp:define-class cypher-union (tree "::utils::Visitable") + ((single-query "SingleQuery *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "SingleQuery")) + (distinct :bool :initval "false" :scope :public) + (union-symbols "std::vector" :scope :public + :documentation "Holds symbols that are created during symbol generation phase. These symbols are used when UNION/UNION ALL combines single query results.")) + (:public + #>cpp + using utils::Visitable::Accept; + + CypherUnion() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + single_query_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit CypherUnion(bool distinct) : distinct_(distinct) {} + CypherUnion(bool distinct, SingleQuery *single_query, + std::vector union_symbols) + : single_query_(single_query), + distinct_(distinct), + union_symbols_(union_symbols) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +(lcp:define-class query (tree "::utils::Visitable>") + () + (:abstractp t) + (:public + #>cpp + using utils::Visitable>::Accept; + + Query() = default; + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk :ignore-other-base-classes t)) + (:clone :ignore-other-base-classes t) + (:type-info :ignore-other-base-classes t)) + +(lcp:define-class cypher-query (query) + ((single-query "SingleQuery *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "SingleQuery") + :documentation "First and potentially only query.") + (cypher-unions "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "CypherUnion") + :documentation "Contains remaining queries that should form and union with `single_query_`.") + (memory-limit "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (memory-scale "size_t" :initval "1024U" :scope :public)) + (:public + #>cpp + CypherQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class explain-query (query) + ((cypher-query "CypherQuery *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "CypherQuery") + :documentation "The CypherQuery to explain.")) + (:public + #>cpp + ExplainQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class profile-query (query) + ((cypher-query "CypherQuery *" + :initval "nullptr" + :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "CypherQuery") + :documentation "The CypherQuery to profile.")) + (:public + #>cpp + ProfileQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class index-query (query) + ((action "Action" :scope :public) + (label "LabelIx" :scope :public + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, storage); + cpp<#) + :clone (lambda (source dest) + #>cpp + ${dest} = storage->GetLabelIx(${source}.name); + cpp<#)) + (properties "std::vector" :scope :public + :slk-load (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + slk::Load(&self->${member}[i], reader, storage); + } + cpp<#) + :clone (clone-name-ix-vector "Property"))) + (:public + (lcp:define-enum action + (create drop) + (:serialize)) + + #>cpp + IndexQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:protected + #>cpp + IndexQuery(Action action, LabelIx label, std::vector properties) + : action_(action), label_(label), properties_(properties) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class create (clause) + ((patterns "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Pattern"))) + (:public + #>cpp + Create() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + for (auto &pattern : patterns_) { + if (!pattern->Accept(visitor)) break; + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit Create(std::vector patterns) : patterns_(patterns) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class call-procedure (clause) + ((procedure-name "std::string" :scope :public) + (arguments "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression")) + (result-fields "std::vector" :scope :public) + (result-identifiers "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Identifier")) + (memory-limit "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (memory-scale "size_t" :initval "1024U" :scope :public) + (is_write :bool :scope :public)) + (:public + #>cpp + CallProcedure() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + bool cont = true; + for (auto &arg : arguments_) { + if (!arg->Accept(visitor)) { + cont = false; + break; + } + } + if (cont) { + for (auto &ident : result_identifiers_) { + if (!ident->Accept(visitor)) { + cont = false; + break; + } + } + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class match (clause) + ((patterns "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Pattern")) + (where "Where *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Where")) + (optional :bool :initval "false" :scope :public)) + (:public + #>cpp + Match() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + bool cont = true; + for (auto &pattern : patterns_) { + if (!pattern->Accept(visitor)) { + cont = false; + break; + } + } + if (cont && where_) { + where_->Accept(visitor); + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit Match(bool optional) : optional_(optional) {} + Match(bool optional, Where *where, std::vector patterns) + : patterns_(patterns), where_(where), optional_(optional) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-enum ordering + (asc desc) + (:documentation "Defines the order for sorting values (ascending or descending).") + (:serialize)) + +(lcp:define-struct sort-item () + ((ordering "Ordering" :scope :public) + (expression "Expression *" + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:serialize (:slk :load-args '((storage "query::v2::AstStorage *")))) + (:clone :args '((storage "AstStorage *")))) + +(lcp:define-struct return-body () + ((distinct :bool :initval "false" + :documentation "True if distinct results should be produced.") + (all-identifiers :bool :initval "false" + :documentation "True if asterisk was found in the return body.") + (named-expressions "std::vector" + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "NamedExpression") + :documentation "Expressions which are used to produce results.") + (order-by "std::vector" + :slk-load (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + slk::Load(&self->${member}[i], reader, storage); + } + cpp<#) + :documentation "Expressions used for ordering the results.") + (skip "Expression *" :initval "nullptr" + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Optional expression on how many results to skip.") + (limit "Expression *" :initval "nullptr" + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Optional expression on how many results to produce.")) + (:documentation "Contents common to @c Return and @c With clauses.") + (:serialize (:slk :load-args '((storage "query::v2::AstStorage *")))) + (:clone :args '((storage "AstStorage *")))) + +(lcp:define-class return (clause) + ((body "ReturnBody" :scope :public + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, storage); + cpp<#))) + (:public + #>cpp + Return() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + bool cont = true; + for (auto &expr : body_.named_expressions) { + if (!expr->Accept(visitor)) { + cont = false; + break; + } + } + if (cont) { + for (auto &order_by : body_.order_by) { + if (!order_by.expression->Accept(visitor)) { + cont = false; + break; + } + } + } + if (cont && body_.skip) cont = body_.skip->Accept(visitor); + if (cont && body_.limit) cont = body_.limit->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit Return(ReturnBody &body) : body_(body) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class with (clause) + ((body "ReturnBody" :scope :public + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, storage); + cpp<#)) + (where "Where *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Where"))) + (:public + #>cpp + With() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + bool cont = true; + for (auto &expr : body_.named_expressions) { + if (!expr->Accept(visitor)) { + cont = false; + break; + } + } + if (cont) { + for (auto &order_by : body_.order_by) { + if (!order_by.expression->Accept(visitor)) { + cont = false; + break; + } + } + } + if (cont && where_) cont = where_->Accept(visitor); + if (cont && body_.skip) cont = body_.skip->Accept(visitor); + if (cont && body_.limit) cont = body_.limit->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + With(ReturnBody &body, Where *where) : body_(body), where_(where) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class delete (clause) + ((expressions "std::vector" + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression") + :scope :public) + (detach :bool :initval "false" :scope :public)) + (:public + #>cpp + Delete() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + for (auto &expr : expressions_) { + if (!expr->Accept(visitor)) break; + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + Delete(bool detach, std::vector expressions) + : expressions_(expressions), detach_(detach) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class set-property (clause) + ((property-lookup "PropertyLookup *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "PropertyLookup")) + (expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:public + #>cpp + SetProperty() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + property_lookup_->Accept(visitor) && expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + SetProperty(PropertyLookup *property_lookup, Expression *expression) + : property_lookup_(property_lookup), expression_(expression) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class set-properties (clause) + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier")) + (expression "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (update :bool :initval "false" :scope :public)) + (:public + #>cpp + SetProperties() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + identifier_->Accept(visitor) && expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + SetProperties(Identifier *identifier, Expression *expression, + bool update = false) + : identifier_(identifier), expression_(expression), update_(update) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class set-labels (clause) + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier")) + (labels "std::vector" :scope :public + :slk-load (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + slk::Load(&self->${member}[i], reader, storage); + } + cpp<#) + :clone (clone-name-ix-vector "Label"))) + (:public + #>cpp + SetLabels() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + identifier_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + SetLabels(Identifier *identifier, const std::vector &labels) + : identifier_(identifier), labels_(labels) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class remove-property (clause) + ((property-lookup "PropertyLookup *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "PropertyLookup"))) + (:public + #>cpp + RemoveProperty() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + property_lookup_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit RemoveProperty(PropertyLookup *property_lookup) + : property_lookup_(property_lookup) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class remove-labels (clause) + ((identifier "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier")) + (labels "std::vector" :scope :public + :slk-load (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + slk::Load(&self->${member}[i], reader, storage); + } + cpp<#) + :clone (clone-name-ix-vector "Label"))) + (:public + #>cpp + RemoveLabels() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + identifier_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + RemoveLabels(Identifier *identifier, const std::vector &labels) + : identifier_(identifier), labels_(labels) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class merge (clause) + ((pattern "Pattern *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Pattern")) + (on-match "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Clause")) + (on-create "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Clause"))) + (:public + #>cpp + Merge() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + bool cont = pattern_->Accept(visitor); + if (cont) { + for (auto &set : on_match_) { + if (!set->Accept(visitor)) { + cont = false; + break; + } + } + } + if (cont) { + for (auto &set : on_create_) { + if (!set->Accept(visitor)) { + cont = false; + break; + } + } + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + Merge(Pattern *pattern, std::vector on_match, + std::vector on_create) + : pattern_(pattern), on_match_(on_match), on_create_(on_create) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class unwind (clause) + ((named-expression "NamedExpression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "NamedExpression"))) + (:public + #>cpp + Unwind() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + named_expression_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit Unwind(NamedExpression *named_expression) + : named_expression_(named_expression) { + DMG_ASSERT(named_expression, "Unwind cannot take nullptr for named_expression"); + } + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class auth-query (query) + ((action "Action" :scope :public) + (user "std::string" :scope :public) + (role "std::string" :scope :public) + (user-or-role "std::string" :scope :public) + (password "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (privileges "std::vector" :scope :public)) + (:public + (lcp:define-enum action + (create-role drop-role show-roles create-user set-password drop-user + show-users set-role clear-role grant-privilege deny-privilege + revoke-privilege show-privileges show-role-for-user + show-users-for-role) + (:serialize)) + (lcp:define-enum privilege + (create delete match merge set remove index stats auth constraint + dump replication durability read_file free_memory trigger config stream module_read module_write + websocket) + (:serialize)) + #>cpp + AuthQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:protected + #>cpp + AuthQuery(Action action, std::string user, std::string role, + std::string user_or_role, Expression *password, + std::vector privileges) + : action_(action), + user_(user), + role_(role), + user_or_role_(user_or_role), + password_(password), + privileges_(privileges) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +;; TODO: Generate this via LCP +#>cpp +/// Constant that holds all available privileges. +const std::vector kPrivilegesAll = { + AuthQuery::Privilege::CREATE, AuthQuery::Privilege::DELETE, + AuthQuery::Privilege::MATCH, AuthQuery::Privilege::MERGE, + AuthQuery::Privilege::SET, AuthQuery::Privilege::REMOVE, + AuthQuery::Privilege::INDEX, AuthQuery::Privilege::STATS, + AuthQuery::Privilege::AUTH, + AuthQuery::Privilege::CONSTRAINT, AuthQuery::Privilege::DUMP, + AuthQuery::Privilege::REPLICATION, + AuthQuery::Privilege::READ_FILE, + AuthQuery::Privilege::DURABILITY, + AuthQuery::Privilege::FREE_MEMORY, AuthQuery::Privilege::TRIGGER, + AuthQuery::Privilege::CONFIG, AuthQuery::Privilege::STREAM, + AuthQuery::Privilege::MODULE_READ, AuthQuery::Privilege::MODULE_WRITE, + AuthQuery::Privilege::WEBSOCKET}; +cpp<# + +(lcp:define-class info-query (query) + ((info-type "InfoType" :scope :public)) + (:public + (lcp:define-enum info-type + (storage index constraint) + (:serialize)) + + #>cpp + DEFVISITABLE(QueryVisitor); + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-struct constraint () + ((type "Type") + (label "LabelIx" :scope :public + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, storage); + cpp<#) + :clone (lambda (source dest) + #>cpp + ${dest} = storage->GetLabelIx(${source}.name); + cpp<#)) + (properties "std::vector" :scope :public + :slk-load (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + slk::Load(&self->${member}[i], reader, storage); + } + cpp<#) + :clone (clone-name-ix-vector "Property"))) + (:public + (lcp:define-enum type (exists unique node-key) + (:serialize (:lcp)))) + (:serialize (:slk :load-args '((storage "query::v2::AstStorage *")))) + (:clone :args '((storage "AstStorage *")))) + +(lcp:define-class constraint-query (query) + ((action-type "ActionType" :scope :public) + (constraint "Constraint" :scope :public + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, storage); + cpp<#))) + (:public + (lcp:define-enum action-type + (create drop) + (:serialize)) + + #>cpp + DEFVISITABLE(QueryVisitor); + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class dump-query (query) () + (:public + #>cpp + DEFVISITABLE(QueryVisitor); + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class replication-query (query) + ((action "Action" :scope :public) + (role "ReplicationRole" :scope :public) + (replica_name "std::string" :scope :public) + (socket_address "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (port "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (sync_mode "SyncMode" :scope :public) + (timeout "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + + (:public + (lcp:define-enum action + (set-replication-role show-replication-role register-replica + drop-replica show-replicas) + (:serialize)) + (lcp:define-enum replication-role + (main replica) + (:serialize)) + (lcp:define-enum sync-mode + (sync async) + (:serialize)) + (lcp:define-enum replica-state + (ready replicating recovery invalid) + (:serialize)) + #>cpp + ReplicationQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class lock-path-query (query) + ((action "Action" :scope :public)) + + (:public + (lcp:define-enum action + (lock-path unlock-path) + (:serialize)) + #>cpp + LockPathQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class load-csv (clause) + ((file "Expression *" :scope :public) + (with_header "bool" :scope :public) + (ignore_bad "bool" :scope :public) + (delimiter "Expression *" :initval "nullptr" :scope :public) + (quote "Expression *" :initval "nullptr" :scope :public) + (row_var "Identifier *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Identifier"))) + + (:public + #>cpp + LoadCsv() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + row_var_->Accept(visitor); + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + explicit LoadCsv(Expression *file, bool with_header, bool ignore_bad, Expression *delimiter, + Expression* quote, Identifier* row_var) + : file_(file), + with_header_(with_header), + ignore_bad_(ignore_bad), + delimiter_(delimiter), + quote_(quote), + row_var_(row_var) { + DMG_ASSERT(row_var, "LoadCsv cannot take nullptr for identifier"); + } + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class free-memory-query (query) () + (:public + #>cpp + DEFVISITABLE(QueryVisitor); + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class trigger-query (query) + ((action "Action" :scope :public) + (event_type "EventType" :scope :public) + (trigger_name "std::string" :scope :public) + (before_commit "bool" :scope :public) + (statement "std::string" :scope :public)) + + (:public + (lcp:define-enum action + (create-trigger drop-trigger show-triggers) + (:serialize)) + (lcp:define-enum event-type + (any vertex_create edge_create create vertex_delete edge_delete delete vertex_update edge_update update) + (:serialize)) + #>cpp + TriggerQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class isolation-level-query (query) + ((isolation_level "IsolationLevel" :scope :public) + (isolation_level_scope "IsolationLevelScope" :scope :public)) + + (:public + (lcp:define-enum isolation-level + (snapshot-isolation read-committed read-uncommitted) + (:serialize)) + (lcp:define-enum isolation-level-scope + (next session global) + (:serialize)) + #>cpp + IsolationLevelQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class create-snapshot-query (query) () + (:public + #>cpp + DEFVISITABLE(QueryVisitor); + cpp<#) + (:serialize (:slk)) + (:clone)) + +(defun clone-variant-topic-names (source destination) + #>cpp + if (auto *topic_expression = std::get_if(&${source})) { + if (*topic_expression == nullptr) { + ${destination} = nullptr; + } else { + ${destination} = (*topic_expression)->Clone(storage); + } + } else { + ${destination} = std::get>(${source}); + } + cpp<#) + +(lcp:define-class stream-query (query) + ((action "Action" :scope :public) + (type "Type" :scope :public) + (stream_name "std::string" :scope :public) + + (batch_limit "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (timeout "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + + (transform_name "std::string" :scope :public) + (batch_interval "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (batch_size "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + + (topic_names "std::variant>" :initval "nullptr" + :clone #'clone-variant-topic-names + :scope :public) + (consumer_group "std::string" :scope :public) + (bootstrap_servers "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + + (service_url "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + + (configs "std::unordered_map" :scope :public + :slk-save #'slk-save-expression-map + :slk-load #'slk-load-expression-map + :clone #'clone-expression-map) + + (credentials "std::unordered_map" :scope :public + :slk-save #'slk-save-expression-map + :slk-load #'slk-load-expression-map + :clone #'clone-expression-map)) + + (:public + (lcp:define-enum action + (create-stream drop-stream start-stream stop-stream start-all-streams stop-all-streams show-streams check-stream) + (:serialize)) + (lcp:define-enum type + (kafka pulsar) + (:serialize)) + #>cpp + StreamQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class setting-query (query) + ((action "Action" :scope :public) + (setting_name "Expression *" :initval "nullptr" :scope :public) + (setting_value "Expression *" :initval "nullptr" :scope :public)) + + (:public + (lcp:define-enum action + (show-setting show-all-settings set-setting) + (:serialize)) + #>cpp + SettingQuery() = default; + + DEFVISITABLE(QueryVisitor); + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class version-query (query) () + (:public + #>cpp + DEFVISITABLE(QueryVisitor); + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class foreach (clause) + ((named_expression "NamedExpression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (clauses "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Clause"))) + (:public + #>cpp + Foreach() = default; + + bool Accept(HierarchicalTreeVisitor &visitor) override { + if (visitor.PreVisit(*this)) { + named_expression_->Accept(visitor); + for (auto &clause : clauses_) { + clause->Accept(visitor); + } + } + return visitor.PostVisit(*this); + } + cpp<#) + (:protected + #>cpp + Foreach(NamedExpression *expression, std::vector clauses) + : named_expression_(expression), clauses_(clauses) {} + cpp<#) + (:private + #>cpp + friend class AstStorage; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:pop-namespace) ;; namespace v2 +(lcp:pop-namespace) ;; namespace query +(lcp:pop-namespace) ;; namespace memgraph diff --git a/src/query/v2/frontend/ast/ast_visitor.hpp b/src/query/v2/frontend/ast/ast_visitor.hpp new file mode 100644 index 000000000..3cd7f9074 --- /dev/null +++ b/src/query/v2/frontend/ast/ast_visitor.hpp @@ -0,0 +1,133 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "utils/visitor.hpp" + +namespace memgraph::query::v2 { + +// Forward declares for Tree visitors. +class CypherQuery; +class SingleQuery; +class CypherUnion; +class NamedExpression; +class Identifier; +class PropertyLookup; +class LabelsTest; +class Aggregation; +class Function; +class Reduce; +class Coalesce; +class Extract; +class All; +class Single; +class Any; +class None; +class ParameterLookup; +class CallProcedure; +class Create; +class Match; +class Return; +class With; +class Pattern; +class NodeAtom; +class EdgeAtom; +class PrimitiveLiteral; +class ListLiteral; +class MapLiteral; +class OrOperator; +class XorOperator; +class AndOperator; +class NotOperator; +class AdditionOperator; +class SubtractionOperator; +class MultiplicationOperator; +class DivisionOperator; +class ModOperator; +class UnaryPlusOperator; +class UnaryMinusOperator; +class IsNullOperator; +class NotEqualOperator; +class EqualOperator; +class LessOperator; +class GreaterOperator; +class LessEqualOperator; +class GreaterEqualOperator; +class InListOperator; +class SubscriptOperator; +class ListSlicingOperator; +class IfOperator; +class Delete; +class Where; +class SetProperty; +class SetProperties; +class SetLabels; +class RemoveProperty; +class RemoveLabels; +class Merge; +class Unwind; +class AuthQuery; +class ExplainQuery; +class ProfileQuery; +class IndexQuery; +class InfoQuery; +class ConstraintQuery; +class RegexMatch; +class DumpQuery; +class ReplicationQuery; +class LockPathQuery; +class LoadCsv; +class FreeMemoryQuery; +class TriggerQuery; +class IsolationLevelQuery; +class CreateSnapshotQuery; +class StreamQuery; +class SettingQuery; +class VersionQuery; +class Foreach; + +using TreeCompositeVisitor = utils::CompositeVisitor< + SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator, + SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator, + LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator, + ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral, + PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None, CallProcedure, + Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels, + RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv, Foreach>; + +using TreeLeafVisitor = utils::LeafVisitor; + +class HierarchicalTreeVisitor : public TreeCompositeVisitor, public TreeLeafVisitor { + public: + using TreeCompositeVisitor::PostVisit; + using TreeCompositeVisitor::PreVisit; + using TreeLeafVisitor::Visit; + using typename TreeLeafVisitor::ReturnType; +}; + +template +class ExpressionVisitor + : public utils::Visitor< + TResult, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator, + SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator, + LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator, + ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, + MapLiteral, PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, + None, ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch> {}; + +template +class QueryVisitor + : public utils::Visitor {}; + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/ast/cypher_main_visitor.cpp b/src/query/v2/frontend/ast/cypher_main_visitor.cpp new file mode 100644 index 000000000..8a74fbbeb --- /dev/null +++ b/src/query/v2/frontend/ast/cypher_main_visitor.cpp @@ -0,0 +1,2362 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/frontend/ast/cypher_main_visitor.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/ast/ast_visitor.hpp" +#include "query/v2/frontend/parsing.hpp" +#include "query/v2/interpret/awesome_memgraph_functions.hpp" +#include "query/v2/procedure/module.hpp" +#include "query/v2/stream/common.hpp" +#include "utils/exceptions.hpp" +#include "utils/logging.hpp" +#include "utils/string.hpp" +#include "utils/typeinfo.hpp" + +namespace memgraph::query::v2::frontend { + +const std::string CypherMainVisitor::kAnonPrefix = "anon"; + +namespace { +template +std::optional> VisitMemoryLimit( + MemgraphCypher::MemoryLimitContext *memory_limit_ctx, TVisitor *visitor) { + MG_ASSERT(memory_limit_ctx); + if (memory_limit_ctx->UNLIMITED()) { + return std::nullopt; + } + + auto memory_limit = memory_limit_ctx->literal()->accept(visitor); + size_t memory_scale = 1024U; + if (memory_limit_ctx->MB()) { + memory_scale = 1024U * 1024U; + } else { + MG_ASSERT(memory_limit_ctx->KB()); + memory_scale = 1024U; + } + + return std::make_pair(memory_limit, memory_scale); +} + +std::string JoinTokens(const auto &tokens, const auto &string_projection, const auto &separator) { + std::vector tokens_string; + tokens_string.reserve(tokens.size()); + for (auto *token : tokens) { + tokens_string.emplace_back(string_projection(token)); + } + return utils::Join(tokens_string, separator); +} + +std::string JoinSymbolicNames(antlr4::tree::ParseTreeVisitor *visitor, + const std::vector symbolicNames, + const std::string &separator = ".") { + return JoinTokens( + symbolicNames, [&](auto *token) { return token->accept(visitor).template as(); }, separator); +} + +std::string JoinSymbolicNamesWithDotsAndMinus(antlr4::tree::ParseTreeVisitor &visitor, + MemgraphCypher::SymbolicNameWithDotsAndMinusContext &ctx) { + return JoinTokens( + ctx.symbolicNameWithMinus(), [&](auto *token) { return JoinSymbolicNames(&visitor, token->symbolicName(), "-"); }, + "."); +} +} // namespace + +antlrcpp::Any CypherMainVisitor::visitExplainQuery(MemgraphCypher::ExplainQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 2, "ExplainQuery should have exactly two children!"); + auto *cypher_query = ctx->children[1]->accept(this).as(); + auto *explain_query = storage_->Create(); + explain_query->cypher_query_ = cypher_query; + query_ = explain_query; + return explain_query; +} + +antlrcpp::Any CypherMainVisitor::visitProfileQuery(MemgraphCypher::ProfileQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 2, "ProfileQuery should have exactly two children!"); + auto *cypher_query = ctx->children[1]->accept(this).as(); + auto *profile_query = storage_->Create(); + profile_query->cypher_query_ = cypher_query; + query_ = profile_query; + return profile_query; +} + +antlrcpp::Any CypherMainVisitor::visitInfoQuery(MemgraphCypher::InfoQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 2, "InfoQuery should have exactly two children!"); + auto *info_query = storage_->Create(); + query_ = info_query; + if (ctx->storageInfo()) { + info_query->info_type_ = InfoQuery::InfoType::STORAGE; + return info_query; + } else if (ctx->indexInfo()) { + info_query->info_type_ = InfoQuery::InfoType::INDEX; + return info_query; + } else if (ctx->constraintInfo()) { + info_query->info_type_ = InfoQuery::InfoType::CONSTRAINT; + return info_query; + } else { + throw utils::NotYetImplemented("Info query: '{}'", ctx->getText()); + } +} + +antlrcpp::Any CypherMainVisitor::visitConstraintQuery(MemgraphCypher::ConstraintQueryContext *ctx) { + auto *constraint_query = storage_->Create(); + MG_ASSERT(ctx->CREATE() || ctx->DROP()); + if (ctx->CREATE()) { + constraint_query->action_type_ = ConstraintQuery::ActionType::CREATE; + } else if (ctx->DROP()) { + constraint_query->action_type_ = ConstraintQuery::ActionType::DROP; + } + constraint_query->constraint_ = ctx->constraint()->accept(this).as(); + query_ = constraint_query; + return query_; +} + +antlrcpp::Any CypherMainVisitor::visitConstraint(MemgraphCypher::ConstraintContext *ctx) { + Constraint constraint; + MG_ASSERT(ctx->EXISTS() || ctx->UNIQUE() || (ctx->NODE() && ctx->KEY())); + if (ctx->EXISTS()) { + constraint.type = Constraint::Type::EXISTS; + } else if (ctx->UNIQUE()) { + constraint.type = Constraint::Type::UNIQUE; + } else if (ctx->NODE() && ctx->KEY()) { + constraint.type = Constraint::Type::NODE_KEY; + } + constraint.label = AddLabel(ctx->labelName()->accept(this)); + std::string node_name = ctx->nodeName->symbolicName()->accept(this); + for (const auto &var_ctx : ctx->constraintPropertyList()->variable()) { + std::string var_name = var_ctx->symbolicName()->accept(this); + if (var_name != node_name) { + throw SemanticException("All constraint variable should reference node '{}'", node_name); + } + } + for (const auto &prop_lookup : ctx->constraintPropertyList()->propertyLookup()) { + constraint.properties.push_back(prop_lookup->propertyKeyName()->accept(this)); + } + + return constraint; +} + +antlrcpp::Any CypherMainVisitor::visitCypherQuery(MemgraphCypher::CypherQueryContext *ctx) { + auto *cypher_query = storage_->Create(); + MG_ASSERT(ctx->singleQuery(), "Expected single query."); + cypher_query->single_query_ = ctx->singleQuery()->accept(this).as(); + + // Check that union and union all dont mix + bool has_union = false; + bool has_union_all = false; + for (auto *child : ctx->cypherUnion()) { + if (child->ALL()) { + has_union_all = true; + } else { + has_union = true; + } + if (has_union && has_union_all) { + throw SemanticException("Invalid combination of UNION and UNION ALL."); + } + cypher_query->cypher_unions_.push_back(child->accept(this).as()); + } + + if (auto *memory_limit_ctx = ctx->queryMemoryLimit()) { + const auto memory_limit_info = VisitMemoryLimit(memory_limit_ctx->memoryLimit(), this); + if (memory_limit_info) { + cypher_query->memory_limit_ = memory_limit_info->first; + cypher_query->memory_scale_ = memory_limit_info->second; + } + } + + query_ = cypher_query; + return cypher_query; +} + +antlrcpp::Any CypherMainVisitor::visitIndexQuery(MemgraphCypher::IndexQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 1, "IndexQuery should have exactly one child!"); + auto *index_query = ctx->children[0]->accept(this).as(); + query_ = index_query; + return index_query; +} + +antlrcpp::Any CypherMainVisitor::visitCreateIndex(MemgraphCypher::CreateIndexContext *ctx) { + auto *index_query = storage_->Create(); + index_query->action_ = IndexQuery::Action::CREATE; + index_query->label_ = AddLabel(ctx->labelName()->accept(this)); + if (ctx->propertyKeyName()) { + PropertyIx name_key = ctx->propertyKeyName()->accept(this); + index_query->properties_ = {name_key}; + } + return index_query; +} + +antlrcpp::Any CypherMainVisitor::visitDropIndex(MemgraphCypher::DropIndexContext *ctx) { + auto *index_query = storage_->Create(); + index_query->action_ = IndexQuery::Action::DROP; + if (ctx->propertyKeyName()) { + PropertyIx key = ctx->propertyKeyName()->accept(this); + index_query->properties_ = {key}; + } + index_query->label_ = AddLabel(ctx->labelName()->accept(this)); + return index_query; +} + +antlrcpp::Any CypherMainVisitor::visitAuthQuery(MemgraphCypher::AuthQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 1, "AuthQuery should have exactly one child!"); + auto *auth_query = ctx->children[0]->accept(this).as(); + query_ = auth_query; + return auth_query; +} + +antlrcpp::Any CypherMainVisitor::visitDumpQuery(MemgraphCypher::DumpQueryContext *ctx) { + auto *dump_query = storage_->Create(); + query_ = dump_query; + return dump_query; +} + +antlrcpp::Any CypherMainVisitor::visitReplicationQuery(MemgraphCypher::ReplicationQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 1, "ReplicationQuery should have exactly one child!"); + auto *replication_query = ctx->children[0]->accept(this).as(); + query_ = replication_query; + return replication_query; +} + +antlrcpp::Any CypherMainVisitor::visitSetReplicationRole(MemgraphCypher::SetReplicationRoleContext *ctx) { + auto *replication_query = storage_->Create(); + replication_query->action_ = ReplicationQuery::Action::SET_REPLICATION_ROLE; + if (ctx->MAIN()) { + if (ctx->WITH() || ctx->PORT()) { + throw SemanticException("Main can't set a port!"); + } + replication_query->role_ = ReplicationQuery::ReplicationRole::MAIN; + } else if (ctx->REPLICA()) { + replication_query->role_ = ReplicationQuery::ReplicationRole::REPLICA; + if (ctx->WITH() && ctx->PORT()) { + if (ctx->port->numberLiteral() && ctx->port->numberLiteral()->integerLiteral()) { + replication_query->port_ = ctx->port->accept(this); + } else { + throw SyntaxException("Port must be an integer literal!"); + } + } + } + return replication_query; +} +antlrcpp::Any CypherMainVisitor::visitShowReplicationRole(MemgraphCypher::ShowReplicationRoleContext *ctx) { + auto *replication_query = storage_->Create(); + replication_query->action_ = ReplicationQuery::Action::SHOW_REPLICATION_ROLE; + return replication_query; +} + +antlrcpp::Any CypherMainVisitor::visitRegisterReplica(MemgraphCypher::RegisterReplicaContext *ctx) { + auto *replication_query = storage_->Create(); + replication_query->action_ = ReplicationQuery::Action::REGISTER_REPLICA; + replication_query->replica_name_ = ctx->replicaName()->symbolicName()->accept(this).as(); + if (ctx->SYNC()) { + replication_query->sync_mode_ = memgraph::query::v2::ReplicationQuery::SyncMode::SYNC; + if (ctx->WITH() && ctx->TIMEOUT()) { + if (ctx->timeout->numberLiteral()) { + // we accept both double and integer literals + replication_query->timeout_ = ctx->timeout->accept(this); + } else { + throw SemanticException("Timeout should be a integer or double literal!"); + } + } + } else if (ctx->ASYNC()) { + if (ctx->WITH() && ctx->TIMEOUT()) { + throw SyntaxException("Timeout can be set only for the SYNC replication mode!"); + } + replication_query->sync_mode_ = memgraph::query::v2::ReplicationQuery::SyncMode::ASYNC; + } + + if (!ctx->socketAddress()->literal()->StringLiteral()) { + throw SemanticException("Socket address should be a string literal!"); + } else { + replication_query->socket_address_ = ctx->socketAddress()->accept(this); + } + + return replication_query; +} + +antlrcpp::Any CypherMainVisitor::visitDropReplica(MemgraphCypher::DropReplicaContext *ctx) { + auto *replication_query = storage_->Create(); + replication_query->action_ = ReplicationQuery::Action::DROP_REPLICA; + replication_query->replica_name_ = ctx->replicaName()->symbolicName()->accept(this).as(); + return replication_query; +} + +antlrcpp::Any CypherMainVisitor::visitShowReplicas(MemgraphCypher::ShowReplicasContext *ctx) { + auto *replication_query = storage_->Create(); + replication_query->action_ = ReplicationQuery::Action::SHOW_REPLICAS; + return replication_query; +} + +antlrcpp::Any CypherMainVisitor::visitLockPathQuery(MemgraphCypher::LockPathQueryContext *ctx) { + auto *lock_query = storage_->Create(); + if (ctx->LOCK()) { + lock_query->action_ = LockPathQuery::Action::LOCK_PATH; + } else if (ctx->UNLOCK()) { + lock_query->action_ = LockPathQuery::Action::UNLOCK_PATH; + } else { + throw SyntaxException("Expected LOCK or UNLOCK"); + } + + query_ = lock_query; + return lock_query; +} + +antlrcpp::Any CypherMainVisitor::visitLoadCsv(MemgraphCypher::LoadCsvContext *ctx) { + query_info_.has_load_csv = true; + + auto *load_csv = storage_->Create(); + // handle file name + if (ctx->csvFile()->literal()->StringLiteral()) { + load_csv->file_ = ctx->csvFile()->accept(this); + } else { + throw SemanticException("CSV file path should be a string literal"); + } + + // handle header options + // Don't have to check for ctx->HEADER(), as it's a mandatory token. + // Just need to check if ctx->WITH() is not nullptr - otherwise, we have a + // ctx->NO() and ctx->HEADER() present. + load_csv->with_header_ = ctx->WITH() != nullptr; + + // handle skip bad row option + load_csv->ignore_bad_ = ctx->IGNORE() && ctx->BAD(); + + // handle delimiter + if (ctx->DELIMITER()) { + if (ctx->delimiter()->literal()->StringLiteral()) { + load_csv->delimiter_ = ctx->delimiter()->accept(this); + } else { + throw SemanticException("Delimiter should be a string literal"); + } + } + + // handle quote + if (ctx->QUOTE()) { + if (ctx->quote()->literal()->StringLiteral()) { + load_csv->quote_ = ctx->quote()->accept(this); + } else { + throw SemanticException("Quote should be a string literal"); + } + } + + // handle row variable + load_csv->row_var_ = storage_->Create(ctx->rowVar()->variable()->accept(this).as()); + + return load_csv; +} + +antlrcpp::Any CypherMainVisitor::visitFreeMemoryQuery(MemgraphCypher::FreeMemoryQueryContext *ctx) { + auto *free_memory_query = storage_->Create(); + query_ = free_memory_query; + return free_memory_query; +} + +antlrcpp::Any CypherMainVisitor::visitTriggerQuery(MemgraphCypher::TriggerQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 1, "TriggerQuery should have exactly one child!"); + auto *trigger_query = ctx->children[0]->accept(this).as(); + query_ = trigger_query; + return trigger_query; +} + +antlrcpp::Any CypherMainVisitor::visitCreateTrigger(MemgraphCypher::CreateTriggerContext *ctx) { + auto *trigger_query = storage_->Create(); + trigger_query->action_ = TriggerQuery::Action::CREATE_TRIGGER; + trigger_query->trigger_name_ = ctx->triggerName()->symbolicName()->accept(this).as(); + + auto *statement = ctx->triggerStatement(); + antlr4::misc::Interval interval{statement->start->getStartIndex(), statement->stop->getStopIndex()}; + trigger_query->statement_ = ctx->start->getInputStream()->getText(interval); + + trigger_query->event_type_ = [ctx] { + if (!ctx->ON()) { + return TriggerQuery::EventType::ANY; + } + + if (ctx->CREATE(1)) { + if (ctx->emptyVertex()) { + return TriggerQuery::EventType::VERTEX_CREATE; + } + if (ctx->emptyEdge()) { + return TriggerQuery::EventType::EDGE_CREATE; + } + return TriggerQuery::EventType::CREATE; + } + + if (ctx->DELETE()) { + if (ctx->emptyVertex()) { + return TriggerQuery::EventType::VERTEX_DELETE; + } + if (ctx->emptyEdge()) { + return TriggerQuery::EventType::EDGE_DELETE; + } + return TriggerQuery::EventType::DELETE; + } + + if (ctx->UPDATE()) { + if (ctx->emptyVertex()) { + return TriggerQuery::EventType::VERTEX_UPDATE; + } + if (ctx->emptyEdge()) { + return TriggerQuery::EventType::EDGE_UPDATE; + } + return TriggerQuery::EventType::UPDATE; + } + + LOG_FATAL("Invalid token allowed for the query"); + }(); + + trigger_query->before_commit_ = ctx->BEFORE(); + + return trigger_query; +} + +antlrcpp::Any CypherMainVisitor::visitDropTrigger(MemgraphCypher::DropTriggerContext *ctx) { + auto *trigger_query = storage_->Create(); + trigger_query->action_ = TriggerQuery::Action::DROP_TRIGGER; + trigger_query->trigger_name_ = ctx->triggerName()->symbolicName()->accept(this).as(); + return trigger_query; +} + +antlrcpp::Any CypherMainVisitor::visitShowTriggers(MemgraphCypher::ShowTriggersContext *ctx) { + auto *trigger_query = storage_->Create(); + trigger_query->action_ = TriggerQuery::Action::SHOW_TRIGGERS; + return trigger_query; +} + +antlrcpp::Any CypherMainVisitor::visitIsolationLevelQuery(MemgraphCypher::IsolationLevelQueryContext *ctx) { + auto *isolation_level_query = storage_->Create(); + + isolation_level_query->isolation_level_scope_ = [scope = ctx->isolationLevelScope()]() { + if (scope->GLOBAL()) { + return IsolationLevelQuery::IsolationLevelScope::GLOBAL; + } + if (scope->SESSION()) { + return IsolationLevelQuery::IsolationLevelScope::SESSION; + } + return IsolationLevelQuery::IsolationLevelScope::NEXT; + }(); + + isolation_level_query->isolation_level_ = [level = ctx->isolationLevel()]() { + if (level->SNAPSHOT()) { + return IsolationLevelQuery::IsolationLevel::SNAPSHOT_ISOLATION; + } + if (level->COMMITTED()) { + return IsolationLevelQuery::IsolationLevel::READ_COMMITTED; + } + return IsolationLevelQuery::IsolationLevel::READ_UNCOMMITTED; + }(); + + query_ = isolation_level_query; + return isolation_level_query; +} + +antlrcpp::Any CypherMainVisitor::visitCreateSnapshotQuery(MemgraphCypher::CreateSnapshotQueryContext *ctx) { + query_ = storage_->Create(); + return query_; +} + +antlrcpp::Any CypherMainVisitor::visitStreamQuery(MemgraphCypher::StreamQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 1, "StreamQuery should have exactly one child!"); + auto *stream_query = ctx->children[0]->accept(this).as(); + query_ = stream_query; + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitCreateStream(MemgraphCypher::CreateStreamContext *ctx) { + MG_ASSERT(ctx->children.size() == 1, "CreateStreamQuery should have exactly one child!"); + auto *stream_query = ctx->children[0]->accept(this).as(); + query_ = stream_query; + return stream_query; +} + +namespace { +std::vector TopicNamesFromSymbols( + antlr4::tree::ParseTreeVisitor &visitor, + const std::vector &topic_name_symbols) { + MG_ASSERT(!topic_name_symbols.empty()); + std::vector topic_names; + topic_names.reserve(topic_name_symbols.size()); + std::transform(topic_name_symbols.begin(), topic_name_symbols.end(), std::back_inserter(topic_names), + [&visitor](auto *topic_name) { return JoinSymbolicNamesWithDotsAndMinus(visitor, *topic_name); }); + return topic_names; +} + +template +concept EnumUint8 = std::is_enum_v && std::same_as>; + +template +void MapConfig(auto &memory, const EnumUint8 auto &enum_key, auto &destination) { + const auto key = static_cast(enum_key); + if (!memory.contains(key)) { + if constexpr (required) { + throw SemanticException("Config {} is required.", ToString(enum_key)); + } else { + return; + } + } + + std::visit( + [&](T &&value) { + using ValueType = std::decay_t; + if constexpr (utils::SameAsAnyOf) { + destination = std::forward(value); + } else { + LOG_FATAL("Invalid type mapped"); + } + }, + std::move(memory[key])); + memory.erase(key); +} + +enum class CommonStreamConfigKey : uint8_t { TRANSFORM, BATCH_INTERVAL, BATCH_SIZE, END }; + +std::string_view ToString(const CommonStreamConfigKey key) { + switch (key) { + case CommonStreamConfigKey::TRANSFORM: + return "TRANSFORM"; + case CommonStreamConfigKey::BATCH_INTERVAL: + return "BATCH_INTERVAL"; + case CommonStreamConfigKey::BATCH_SIZE: + return "BATCH_SIZE"; + case CommonStreamConfigKey::END: + LOG_FATAL("Invalid config key used"); + } +} + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define GENERATE_STREAM_CONFIG_KEY_ENUM(stream, first_config, ...) \ + enum class BOOST_PP_CAT(stream, ConfigKey) : uint8_t { \ + first_config = static_cast(CommonStreamConfigKey::END), \ + __VA_ARGS__ \ + }; + +GENERATE_STREAM_CONFIG_KEY_ENUM(Kafka, TOPICS, CONSUMER_GROUP, BOOTSTRAP_SERVERS, CONFIGS, CREDENTIALS); + +std::string_view ToString(const KafkaConfigKey key) { + switch (key) { + case KafkaConfigKey::TOPICS: + return "TOPICS"; + case KafkaConfigKey::CONSUMER_GROUP: + return "CONSUMER_GROUP"; + case KafkaConfigKey::BOOTSTRAP_SERVERS: + return "BOOTSTRAP_SERVERS"; + case KafkaConfigKey::CONFIGS: + return "CONFIGS"; + case KafkaConfigKey::CREDENTIALS: + return "CREDENTIALS"; + } +} + +void MapCommonStreamConfigs(auto &memory, StreamQuery &stream_query) { + MapConfig(memory, CommonStreamConfigKey::TRANSFORM, stream_query.transform_name_); + MapConfig(memory, CommonStreamConfigKey::BATCH_INTERVAL, stream_query.batch_interval_); + MapConfig(memory, CommonStreamConfigKey::BATCH_SIZE, stream_query.batch_size_); +} +} // namespace + +antlrcpp::Any CypherMainVisitor::visitConfigKeyValuePair(MemgraphCypher::ConfigKeyValuePairContext *ctx) { + MG_ASSERT(ctx->literal().size() == 2); + return std::pair{ctx->literal(0)->accept(this).as(), ctx->literal(1)->accept(this).as()}; +} + +antlrcpp::Any CypherMainVisitor::visitConfigMap(MemgraphCypher::ConfigMapContext *ctx) { + std::unordered_map map; + for (auto *key_value_pair : ctx->configKeyValuePair()) { + // If the queries are cached, then only the stripped query is parsed, so the actual keys cannot be determined + // here. That means duplicates cannot be checked. + map.insert(key_value_pair->accept(this).as>()); + } + return map; +} + +antlrcpp::Any CypherMainVisitor::visitKafkaCreateStream(MemgraphCypher::KafkaCreateStreamContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::CREATE_STREAM; + stream_query->type_ = StreamQuery::Type::KAFKA; + stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as(); + + for (auto *create_config_ctx : ctx->kafkaCreateStreamConfig()) { + create_config_ctx->accept(this); + } + + MapConfig, Expression *>(memory_, KafkaConfigKey::TOPICS, stream_query->topic_names_); + MapConfig(memory_, KafkaConfigKey::CONSUMER_GROUP, stream_query->consumer_group_); + MapConfig(memory_, KafkaConfigKey::BOOTSTRAP_SERVERS, stream_query->bootstrap_servers_); + MapConfig>(memory_, KafkaConfigKey::CONFIGS, + stream_query->configs_); + MapConfig>(memory_, KafkaConfigKey::CREDENTIALS, + stream_query->credentials_); + + MapCommonStreamConfigs(memory_, *stream_query); + + return stream_query; +} + +namespace { +void ThrowIfExists(const auto &map, const EnumUint8 auto &enum_key) { + const auto key = static_cast(enum_key); + if (map.contains(key)) { + throw SemanticException("{} defined multiple times in the query", ToString(enum_key)); + } +} + +void GetTopicNames(auto &destination, MemgraphCypher::TopicNamesContext *topic_names_ctx, + antlr4::tree::ParseTreeVisitor &visitor) { + MG_ASSERT(topic_names_ctx != nullptr); + if (auto *symbolic_topic_names_ctx = topic_names_ctx->symbolicTopicNames()) { + destination = TopicNamesFromSymbols(visitor, symbolic_topic_names_ctx->symbolicNameWithDotsAndMinus()); + } else { + if (!topic_names_ctx->literal()->StringLiteral()) { + throw SemanticException("Topic names should be defined as a string literal or as symbolic names"); + } + destination = topic_names_ctx->accept(&visitor).as(); + } +} +} // namespace + +antlrcpp::Any CypherMainVisitor::visitKafkaCreateStreamConfig(MemgraphCypher::KafkaCreateStreamConfigContext *ctx) { + if (ctx->commonCreateStreamConfig()) { + return ctx->commonCreateStreamConfig()->accept(this); + } + + if (ctx->TOPICS()) { + ThrowIfExists(memory_, KafkaConfigKey::TOPICS); + static constexpr auto topics_key = static_cast(KafkaConfigKey::TOPICS); + GetTopicNames(memory_[topics_key], ctx->topicNames(), *this); + return {}; + } + + if (ctx->CONSUMER_GROUP()) { + ThrowIfExists(memory_, KafkaConfigKey::CONSUMER_GROUP); + static constexpr auto consumer_group_key = static_cast(KafkaConfigKey::CONSUMER_GROUP); + memory_[consumer_group_key] = JoinSymbolicNamesWithDotsAndMinus(*this, *ctx->consumerGroup); + return {}; + } + + if (ctx->CONFIGS()) { + ThrowIfExists(memory_, KafkaConfigKey::CONFIGS); + static constexpr auto configs_key = static_cast(KafkaConfigKey::CONFIGS); + memory_.emplace(configs_key, ctx->configsMap->accept(this).as>()); + return {}; + } + + if (ctx->CREDENTIALS()) { + ThrowIfExists(memory_, KafkaConfigKey::CREDENTIALS); + static constexpr auto credentials_key = static_cast(KafkaConfigKey::CREDENTIALS); + memory_.emplace(credentials_key, + ctx->credentialsMap->accept(this).as>()); + return {}; + } + + MG_ASSERT(ctx->BOOTSTRAP_SERVERS()); + ThrowIfExists(memory_, KafkaConfigKey::BOOTSTRAP_SERVERS); + if (!ctx->bootstrapServers->StringLiteral()) { + throw SemanticException("Bootstrap servers should be a string!"); + } + + const auto bootstrap_servers_key = static_cast(KafkaConfigKey::BOOTSTRAP_SERVERS); + memory_[bootstrap_servers_key] = ctx->bootstrapServers->accept(this).as(); + return {}; +} + +namespace { +GENERATE_STREAM_CONFIG_KEY_ENUM(Pulsar, TOPICS, SERVICE_URL); + +std::string_view ToString(const PulsarConfigKey key) { + switch (key) { + case PulsarConfigKey::TOPICS: + return "TOPICS"; + case PulsarConfigKey::SERVICE_URL: + return "SERVICE_URL"; + } +} +} // namespace + +antlrcpp::Any CypherMainVisitor::visitPulsarCreateStream(MemgraphCypher::PulsarCreateStreamContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::CREATE_STREAM; + stream_query->type_ = StreamQuery::Type::PULSAR; + stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as(); + + for (auto *create_config_ctx : ctx->pulsarCreateStreamConfig()) { + create_config_ctx->accept(this); + } + + MapConfig, Expression *>(memory_, PulsarConfigKey::TOPICS, stream_query->topic_names_); + MapConfig(memory_, PulsarConfigKey::SERVICE_URL, stream_query->service_url_); + + MapCommonStreamConfigs(memory_, *stream_query); + + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitPulsarCreateStreamConfig(MemgraphCypher::PulsarCreateStreamConfigContext *ctx) { + if (ctx->commonCreateStreamConfig()) { + return ctx->commonCreateStreamConfig()->accept(this); + } + + if (ctx->TOPICS()) { + ThrowIfExists(memory_, PulsarConfigKey::TOPICS); + const auto topics_key = static_cast(PulsarConfigKey::TOPICS); + GetTopicNames(memory_[topics_key], ctx->topicNames(), *this); + return {}; + } + + MG_ASSERT(ctx->SERVICE_URL()); + ThrowIfExists(memory_, PulsarConfigKey::SERVICE_URL); + if (!ctx->serviceUrl->StringLiteral()) { + throw SemanticException("Service URL must be a string!"); + } + const auto service_url_key = static_cast(PulsarConfigKey::SERVICE_URL); + memory_[service_url_key] = ctx->serviceUrl->accept(this).as(); + return {}; +} + +antlrcpp::Any CypherMainVisitor::visitCommonCreateStreamConfig(MemgraphCypher::CommonCreateStreamConfigContext *ctx) { + if (ctx->TRANSFORM()) { + ThrowIfExists(memory_, CommonStreamConfigKey::TRANSFORM); + const auto transform_key = static_cast(CommonStreamConfigKey::TRANSFORM); + memory_[transform_key] = JoinSymbolicNames(this, ctx->transformationName->symbolicName()); + return {}; + } + + if (ctx->BATCH_INTERVAL()) { + ThrowIfExists(memory_, CommonStreamConfigKey::BATCH_INTERVAL); + if (!ctx->batchInterval->numberLiteral() || !ctx->batchInterval->numberLiteral()->integerLiteral()) { + throw SemanticException("Batch interval must be an integer literal!"); + } + const auto batch_interval_key = static_cast(CommonStreamConfigKey::BATCH_INTERVAL); + memory_[batch_interval_key] = ctx->batchInterval->accept(this).as(); + return {}; + } + + MG_ASSERT(ctx->BATCH_SIZE()); + ThrowIfExists(memory_, CommonStreamConfigKey::BATCH_SIZE); + if (!ctx->batchSize->numberLiteral() || !ctx->batchSize->numberLiteral()->integerLiteral()) { + throw SemanticException("Batch size must be an integer literal!"); + } + const auto batch_size_key = static_cast(CommonStreamConfigKey::BATCH_SIZE); + memory_[batch_size_key] = ctx->batchSize->accept(this).as(); + return {}; +} + +antlrcpp::Any CypherMainVisitor::visitDropStream(MemgraphCypher::DropStreamContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::DROP_STREAM; + stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as(); + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitStartStream(MemgraphCypher::StartStreamContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::START_STREAM; + + if (ctx->BATCH_LIMIT()) { + if (!ctx->batchLimit->numberLiteral() || !ctx->batchLimit->numberLiteral()->integerLiteral()) { + throw SemanticException("Batch limit should be an integer literal!"); + } + stream_query->batch_limit_ = ctx->batchLimit->accept(this); + } + if (ctx->TIMEOUT()) { + if (!ctx->timeout->numberLiteral() || !ctx->timeout->numberLiteral()->integerLiteral()) { + throw SemanticException("Timeout should be an integer literal!"); + } + if (!ctx->BATCH_LIMIT()) { + throw SemanticException("Parameter TIMEOUT can only be defined if BATCH_LIMIT is defined"); + } + stream_query->timeout_ = ctx->timeout->accept(this); + } + + stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as(); + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitStartAllStreams(MemgraphCypher::StartAllStreamsContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::START_ALL_STREAMS; + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitStopStream(MemgraphCypher::StopStreamContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::STOP_STREAM; + stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as(); + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitStopAllStreams(MemgraphCypher::StopAllStreamsContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::STOP_ALL_STREAMS; + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitShowStreams(MemgraphCypher::ShowStreamsContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::SHOW_STREAMS; + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitCheckStream(MemgraphCypher::CheckStreamContext *ctx) { + auto *stream_query = storage_->Create(); + stream_query->action_ = StreamQuery::Action::CHECK_STREAM; + stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as(); + + if (ctx->BATCH_LIMIT()) { + if (!ctx->batchLimit->numberLiteral() || !ctx->batchLimit->numberLiteral()->integerLiteral()) { + throw SemanticException("Batch limit should be an integer literal!"); + } + stream_query->batch_limit_ = ctx->batchLimit->accept(this); + } + if (ctx->TIMEOUT()) { + if (!ctx->timeout->numberLiteral() || !ctx->timeout->numberLiteral()->integerLiteral()) { + throw SemanticException("Timeout should be an integer literal!"); + } + stream_query->timeout_ = ctx->timeout->accept(this); + } + return stream_query; +} + +antlrcpp::Any CypherMainVisitor::visitSettingQuery(MemgraphCypher::SettingQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 1, "SettingQuery should have exactly one child!"); + auto *setting_query = ctx->children[0]->accept(this).as(); + query_ = setting_query; + return setting_query; +} + +antlrcpp::Any CypherMainVisitor::visitSetSetting(MemgraphCypher::SetSettingContext *ctx) { + auto *setting_query = storage_->Create(); + setting_query->action_ = SettingQuery::Action::SET_SETTING; + + if (!ctx->settingName()->literal()->StringLiteral()) { + throw SemanticException("Setting name should be a string literal"); + } + + if (!ctx->settingValue()->literal()->StringLiteral()) { + throw SemanticException("Setting value should be a string literal"); + } + + setting_query->setting_name_ = ctx->settingName()->accept(this); + MG_ASSERT(setting_query->setting_name_); + + setting_query->setting_value_ = ctx->settingValue()->accept(this); + MG_ASSERT(setting_query->setting_value_); + return setting_query; +} + +antlrcpp::Any CypherMainVisitor::visitShowSetting(MemgraphCypher::ShowSettingContext *ctx) { + auto *setting_query = storage_->Create(); + setting_query->action_ = SettingQuery::Action::SHOW_SETTING; + + if (!ctx->settingName()->literal()->StringLiteral()) { + throw SemanticException("Setting name should be a string literal"); + } + + setting_query->setting_name_ = ctx->settingName()->accept(this); + MG_ASSERT(setting_query->setting_name_); + + return setting_query; +} + +antlrcpp::Any CypherMainVisitor::visitShowSettings(MemgraphCypher::ShowSettingsContext * /*ctx*/) { + auto *setting_query = storage_->Create(); + setting_query->action_ = SettingQuery::Action::SHOW_ALL_SETTINGS; + return setting_query; +} + +antlrcpp::Any CypherMainVisitor::visitVersionQuery(MemgraphCypher::VersionQueryContext * /*ctx*/) { + auto *version_query = storage_->Create(); + query_ = version_query; + return version_query; +} + +antlrcpp::Any CypherMainVisitor::visitCypherUnion(MemgraphCypher::CypherUnionContext *ctx) { + bool distinct = !ctx->ALL(); + auto *cypher_union = storage_->Create(distinct); + DMG_ASSERT(ctx->singleQuery(), "Expected single query."); + cypher_union->single_query_ = ctx->singleQuery()->accept(this).as(); + return cypher_union; +} + +antlrcpp::Any CypherMainVisitor::visitSingleQuery(MemgraphCypher::SingleQueryContext *ctx) { + auto *single_query = storage_->Create(); + for (auto *child : ctx->clause()) { + antlrcpp::Any got = child->accept(this); + if (got.is()) { + single_query->clauses_.push_back(got.as()); + } else { + auto child_clauses = got.as>(); + single_query->clauses_.insert(single_query->clauses_.end(), child_clauses.begin(), child_clauses.end()); + } + } + + // Check if ordering of clauses makes sense. + // + // TODO: should we forbid multiple consecutive set clauses? That case is + // little bit problematic because multiple barriers are needed. Multiple + // consecutive SET clauses are undefined behaviour in neo4j. + bool has_update = false; + bool has_return = false; + bool has_optional_match = false; + bool has_call_procedure = false; + bool calls_write_procedure = false; + bool has_any_update = false; + bool has_load_csv = false; + + auto check_write_procedure = [&calls_write_procedure](const std::string_view clause) { + if (calls_write_procedure) { + throw SemanticException( + "{} can't be put after calling a writeable procedure, only RETURN clause can be put after.", clause); + } + }; + + for (Clause *clause : single_query->clauses_) { + const auto &clause_type = clause->GetTypeInfo(); + if (const auto *call_procedure = utils::Downcast(clause); call_procedure != nullptr) { + if (has_return) { + throw SemanticException("CALL can't be put after RETURN clause."); + } + check_write_procedure("CALL"); + has_call_procedure = true; + if (call_procedure->is_write_) { + calls_write_procedure = true; + has_update = true; + } + } else if (utils::IsSubtype(clause_type, Unwind::kType)) { + check_write_procedure("UNWIND"); + if (has_update || has_return) { + throw SemanticException("UNWIND can't be put after RETURN clause or after an update."); + } + } else if (utils::IsSubtype(clause_type, LoadCsv::kType)) { + if (has_load_csv) { + throw SemanticException("Can't have multiple LOAD CSV clauses in a single query."); + } + check_write_procedure("LOAD CSV"); + if (has_return) { + throw SemanticException("LOAD CSV can't be put after RETURN clause."); + } + has_load_csv = true; + } else if (auto *match = utils::Downcast(clause)) { + if (has_update || has_return) { + throw SemanticException("MATCH can't be put after RETURN clause or after an update."); + } + if (match->optional_) { + has_optional_match = true; + } else if (has_optional_match) { + throw SemanticException("MATCH can't be put after OPTIONAL MATCH."); + } + check_write_procedure("MATCH"); + } else if (utils::IsSubtype(clause_type, Create::kType) || utils::IsSubtype(clause_type, Delete::kType) || + utils::IsSubtype(clause_type, SetProperty::kType) || + utils::IsSubtype(clause_type, SetProperties::kType) || utils::IsSubtype(clause_type, SetLabels::kType) || + utils::IsSubtype(clause_type, RemoveProperty::kType) || + utils::IsSubtype(clause_type, RemoveLabels::kType) || utils::IsSubtype(clause_type, Merge::kType) || + utils::IsSubtype(clause_type, Foreach::kType)) { + if (has_return) { + throw SemanticException("Update clause can't be used after RETURN."); + } + check_write_procedure("Update clause"); + has_update = true; + has_any_update = true; + } else if (utils::IsSubtype(clause_type, Return::kType)) { + if (has_return) { + throw SemanticException("There can only be one RETURN in a clause."); + } + has_return = true; + } else if (utils::IsSubtype(clause_type, With::kType)) { + if (has_return) { + throw SemanticException("RETURN can't be put before WITH."); + } + check_write_procedure("WITH"); + has_update = has_return = has_optional_match = false; + } else { + DLOG_FATAL("Can't happen"); + } + } + bool is_standalone_call_procedure = has_call_procedure && single_query->clauses_.size() == 1U; + if (!has_update && !has_return && !is_standalone_call_procedure) { + throw SemanticException("Query should either create or update something, or return results!"); + } + + if (has_any_update && calls_write_procedure) { + throw SemanticException("Write procedures cannot be used in queries that contains any update clauses!"); + } + // Construct unique names for anonymous identifiers; + int id = 1; + for (auto **identifier : anonymous_identifiers) { + while (true) { + std::string id_name = kAnonPrefix + std::to_string(id++); + if (users_identifiers.find(id_name) == users_identifiers.end()) { + *identifier = storage_->Create(id_name, false); + break; + } + } + } + return single_query; +} + +antlrcpp::Any CypherMainVisitor::visitClause(MemgraphCypher::ClauseContext *ctx) { + if (ctx->cypherReturn()) { + return static_cast(ctx->cypherReturn()->accept(this).as()); + } + if (ctx->cypherMatch()) { + return static_cast(ctx->cypherMatch()->accept(this).as()); + } + if (ctx->create()) { + return static_cast(ctx->create()->accept(this).as()); + } + if (ctx->cypherDelete()) { + return static_cast(ctx->cypherDelete()->accept(this).as()); + } + if (ctx->set()) { + // Different return type!!! + return ctx->set()->accept(this).as>(); + } + if (ctx->remove()) { + // Different return type!!! + return ctx->remove()->accept(this).as>(); + } + if (ctx->with()) { + return static_cast(ctx->with()->accept(this).as()); + } + if (ctx->merge()) { + return static_cast(ctx->merge()->accept(this).as()); + } + if (ctx->unwind()) { + return static_cast(ctx->unwind()->accept(this).as()); + } + if (ctx->callProcedure()) { + return static_cast(ctx->callProcedure()->accept(this).as()); + } + if (ctx->loadCsv()) { + return static_cast(ctx->loadCsv()->accept(this).as()); + } + if (ctx->foreach ()) { + return static_cast(ctx->foreach ()->accept(this).as()); + } + // TODO: implement other clauses. + throw utils::NotYetImplemented("clause '{}'", ctx->getText()); + return 0; +} + +antlrcpp::Any CypherMainVisitor::visitCypherMatch(MemgraphCypher::CypherMatchContext *ctx) { + auto *match = storage_->Create(); + match->optional_ = !!ctx->OPTIONAL(); + if (ctx->where()) { + match->where_ = ctx->where()->accept(this); + } + match->patterns_ = ctx->pattern()->accept(this).as>(); + return match; +} + +antlrcpp::Any CypherMainVisitor::visitCreate(MemgraphCypher::CreateContext *ctx) { + auto *create = storage_->Create(); + create->patterns_ = ctx->pattern()->accept(this).as>(); + return create; +} + +antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedureContext *ctx) { + // Don't cache queries which call procedures because the + // procedure definition can affect the behaviour of the visitor and + // the execution of the query. + // If a user recompiles and reloads the procedure with different result + // names, because of the cache, old result names will be expected while the + // procedure will return results mapped to new names. + query_info_.is_cacheable = false; + + auto *call_proc = storage_->Create(); + MG_ASSERT(!ctx->procedureName()->symbolicName().empty()); + call_proc->procedure_name_ = JoinSymbolicNames(this, ctx->procedureName()->symbolicName()); + call_proc->arguments_.reserve(ctx->expression().size()); + for (auto *expr : ctx->expression()) { + call_proc->arguments_.push_back(expr->accept(this)); + } + + if (auto *memory_limit_ctx = ctx->procedureMemoryLimit()) { + const auto memory_limit_info = VisitMemoryLimit(memory_limit_ctx->memoryLimit(), this); + if (memory_limit_info) { + call_proc->memory_limit_ = memory_limit_info->first; + call_proc->memory_scale_ = memory_limit_info->second; + } + } else { + // Default to 100 MB + call_proc->memory_limit_ = storage_->Create(TypedValue(100)); + call_proc->memory_scale_ = 1024U * 1024U; + } + + const auto &maybe_found = + procedure::FindProcedure(procedure::gModuleRegistry, call_proc->procedure_name_, utils::NewDeleteResource()); + if (!maybe_found) { + throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_); + } + call_proc->is_write_ = maybe_found->second->info.is_write; + + auto *yield_ctx = ctx->yieldProcedureResults(); + if (!yield_ctx) { + if (!maybe_found->second->results.empty()) { + throw SemanticException( + "CALL without YIELD may only be used on procedures which do not " + "return any result fields."); + } + // When we return, we will release the lock on modules. This means that + // someone may reload the procedure and change the result signature. But to + // keep the implementation simple, we ignore the case as the rest of the + // code doesn't really care whether we yield or not, so it should not break. + return call_proc; + } + if (yield_ctx->getTokens(MemgraphCypher::ASTERISK).empty()) { + call_proc->result_fields_.reserve(yield_ctx->procedureResult().size()); + call_proc->result_identifiers_.reserve(yield_ctx->procedureResult().size()); + for (auto *result : yield_ctx->procedureResult()) { + MG_ASSERT(result->variable().size() == 1 || result->variable().size() == 2); + call_proc->result_fields_.push_back(result->variable()[0]->accept(this).as()); + std::string result_alias; + if (result->variable().size() == 2) { + result_alias = result->variable()[1]->accept(this).as(); + } else { + result_alias = result->variable()[0]->accept(this).as(); + } + call_proc->result_identifiers_.push_back(storage_->Create(result_alias)); + } + } else { + const auto &maybe_found = + procedure::FindProcedure(procedure::gModuleRegistry, call_proc->procedure_name_, utils::NewDeleteResource()); + if (!maybe_found) { + throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_); + } + const auto &[module, proc] = *maybe_found; + call_proc->result_fields_.reserve(proc->results.size()); + call_proc->result_identifiers_.reserve(proc->results.size()); + for (const auto &[result_name, desc] : proc->results) { + bool is_deprecated = desc.second; + if (is_deprecated) continue; + call_proc->result_fields_.emplace_back(result_name); + call_proc->result_identifiers_.push_back(storage_->Create(std::string(result_name))); + } + // When we leave the scope, we will release the lock on modules. This means + // that someone may reload the procedure and change its result signature. We + // are fine with this, because if new result fields were added then we yield + // the subset of those and that will appear to a user as if they used the + // procedure before reload. Any subsequent `CALL ... YIELD *` will fetch the + // new fields as well. In case the result signature has had some result + // fields removed, then the query execution will report an error that we are + // yielding missing fields. The user can then just retry the query. + } + + return call_proc; +} + +/** + * @return std::string + */ +antlrcpp::Any CypherMainVisitor::visitUserOrRoleName(MemgraphCypher::UserOrRoleNameContext *ctx) { + return ctx->symbolicName()->accept(this).as(); +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitCreateRole(MemgraphCypher::CreateRoleContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::CREATE_ROLE; + auth->role_ = ctx->role->accept(this).as(); + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitDropRole(MemgraphCypher::DropRoleContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::DROP_ROLE; + auth->role_ = ctx->role->accept(this).as(); + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitShowRoles(MemgraphCypher::ShowRolesContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::SHOW_ROLES; + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitCreateUser(MemgraphCypher::CreateUserContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::CREATE_USER; + auth->user_ = ctx->user->accept(this).as(); + if (ctx->password) { + if (!ctx->password->StringLiteral() && !ctx->literal()->CYPHERNULL()) { + throw SyntaxException("Password should be a string literal or null."); + } + auth->password_ = ctx->password->accept(this); + } + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitSetPassword(MemgraphCypher::SetPasswordContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::SET_PASSWORD; + auth->user_ = ctx->user->accept(this).as(); + if (!ctx->password->StringLiteral() && !ctx->literal()->CYPHERNULL()) { + throw SyntaxException("Password should be a string literal or null."); + } + auth->password_ = ctx->password->accept(this); + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitDropUser(MemgraphCypher::DropUserContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::DROP_USER; + auth->user_ = ctx->user->accept(this).as(); + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitShowUsers(MemgraphCypher::ShowUsersContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::SHOW_USERS; + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitSetRole(MemgraphCypher::SetRoleContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::SET_ROLE; + auth->user_ = ctx->user->accept(this).as(); + auth->role_ = ctx->role->accept(this).as(); + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitClearRole(MemgraphCypher::ClearRoleContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::CLEAR_ROLE; + auth->user_ = ctx->user->accept(this).as(); + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitGrantPrivilege(MemgraphCypher::GrantPrivilegeContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::GRANT_PRIVILEGE; + auth->user_or_role_ = ctx->userOrRole->accept(this).as(); + if (ctx->privilegeList()) { + for (auto *privilege : ctx->privilegeList()->privilege()) { + auth->privileges_.push_back(privilege->accept(this)); + } + } else { + /* grant all privileges */ + auth->privileges_ = kPrivilegesAll; + } + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitDenyPrivilege(MemgraphCypher::DenyPrivilegeContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::DENY_PRIVILEGE; + auth->user_or_role_ = ctx->userOrRole->accept(this).as(); + if (ctx->privilegeList()) { + for (auto *privilege : ctx->privilegeList()->privilege()) { + auth->privileges_.push_back(privilege->accept(this)); + } + } else { + /* deny all privileges */ + auth->privileges_ = kPrivilegesAll; + } + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitRevokePrivilege(MemgraphCypher::RevokePrivilegeContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::REVOKE_PRIVILEGE; + auth->user_or_role_ = ctx->userOrRole->accept(this).as(); + if (ctx->privilegeList()) { + for (auto *privilege : ctx->privilegeList()->privilege()) { + auth->privileges_.push_back(privilege->accept(this)); + } + } else { + /* revoke all privileges */ + auth->privileges_ = kPrivilegesAll; + } + return auth; +} + +/** + * @return AuthQuery::Privilege + */ +antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext *ctx) { + if (ctx->CREATE()) return AuthQuery::Privilege::CREATE; + if (ctx->DELETE()) return AuthQuery::Privilege::DELETE; + if (ctx->MATCH()) return AuthQuery::Privilege::MATCH; + if (ctx->MERGE()) return AuthQuery::Privilege::MERGE; + if (ctx->SET()) return AuthQuery::Privilege::SET; + if (ctx->REMOVE()) return AuthQuery::Privilege::REMOVE; + if (ctx->INDEX()) return AuthQuery::Privilege::INDEX; + if (ctx->STATS()) return AuthQuery::Privilege::STATS; + if (ctx->AUTH()) return AuthQuery::Privilege::AUTH; + if (ctx->CONSTRAINT()) return AuthQuery::Privilege::CONSTRAINT; + if (ctx->DUMP()) return AuthQuery::Privilege::DUMP; + if (ctx->REPLICATION()) return AuthQuery::Privilege::REPLICATION; + if (ctx->READ_FILE()) return AuthQuery::Privilege::READ_FILE; + if (ctx->FREE_MEMORY()) return AuthQuery::Privilege::FREE_MEMORY; + if (ctx->TRIGGER()) return AuthQuery::Privilege::TRIGGER; + if (ctx->CONFIG()) return AuthQuery::Privilege::CONFIG; + if (ctx->DURABILITY()) return AuthQuery::Privilege::DURABILITY; + if (ctx->STREAM()) return AuthQuery::Privilege::STREAM; + if (ctx->MODULE_READ()) return AuthQuery::Privilege::MODULE_READ; + if (ctx->MODULE_WRITE()) return AuthQuery::Privilege::MODULE_WRITE; + if (ctx->WEBSOCKET()) return AuthQuery::Privilege::WEBSOCKET; + LOG_FATAL("Should not get here - unknown privilege!"); +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitShowPrivileges(MemgraphCypher::ShowPrivilegesContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::SHOW_PRIVILEGES; + auth->user_or_role_ = ctx->userOrRole->accept(this).as(); + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitShowRoleForUser(MemgraphCypher::ShowRoleForUserContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::SHOW_ROLE_FOR_USER; + auth->user_ = ctx->user->accept(this).as(); + return auth; +} + +/** + * @return AuthQuery* + */ +antlrcpp::Any CypherMainVisitor::visitShowUsersForRole(MemgraphCypher::ShowUsersForRoleContext *ctx) { + AuthQuery *auth = storage_->Create(); + auth->action_ = AuthQuery::Action::SHOW_USERS_FOR_ROLE; + auth->role_ = ctx->role->accept(this).as(); + return auth; +} + +antlrcpp::Any CypherMainVisitor::visitCypherReturn(MemgraphCypher::CypherReturnContext *ctx) { + auto *return_clause = storage_->Create(); + return_clause->body_ = ctx->returnBody()->accept(this); + if (ctx->DISTINCT()) { + return_clause->body_.distinct = true; + } + return return_clause; +} + +antlrcpp::Any CypherMainVisitor::visitReturnBody(MemgraphCypher::ReturnBodyContext *ctx) { + ReturnBody body; + if (ctx->order()) { + body.order_by = ctx->order()->accept(this).as>(); + } + if (ctx->skip()) { + body.skip = static_cast(ctx->skip()->accept(this)); + } + if (ctx->limit()) { + body.limit = static_cast(ctx->limit()->accept(this)); + } + std::tie(body.all_identifiers, body.named_expressions) = + ctx->returnItems()->accept(this).as>>(); + return body; +} + +antlrcpp::Any CypherMainVisitor::visitReturnItems(MemgraphCypher::ReturnItemsContext *ctx) { + std::vector named_expressions; + for (auto *item : ctx->returnItem()) { + named_expressions.push_back(item->accept(this)); + } + return std::pair>(ctx->getTokens(MemgraphCypher::ASTERISK).size(), + named_expressions); +} + +antlrcpp::Any CypherMainVisitor::visitReturnItem(MemgraphCypher::ReturnItemContext *ctx) { + auto *named_expr = storage_->Create(); + named_expr->expression_ = ctx->expression()->accept(this); + MG_ASSERT(named_expr->expression_); + if (ctx->variable()) { + named_expr->name_ = std::string(ctx->variable()->accept(this).as()); + users_identifiers.insert(named_expr->name_); + } else { + if (in_with_ && !utils::IsSubtype(*named_expr->expression_, Identifier::kType)) { + throw SemanticException("Only variables can be non-aliased in WITH."); + } + named_expr->name_ = std::string(ctx->getText()); + named_expr->token_position_ = ctx->expression()->getStart()->getTokenIndex(); + } + return named_expr; +} + +antlrcpp::Any CypherMainVisitor::visitOrder(MemgraphCypher::OrderContext *ctx) { + std::vector order_by; + for (auto *sort_item : ctx->sortItem()) { + order_by.push_back(sort_item->accept(this)); + } + return order_by; +} + +antlrcpp::Any CypherMainVisitor::visitSortItem(MemgraphCypher::SortItemContext *ctx) { + return SortItem{ctx->DESC() || ctx->DESCENDING() ? Ordering::DESC : Ordering::ASC, ctx->expression()->accept(this)}; +} + +antlrcpp::Any CypherMainVisitor::visitNodePattern(MemgraphCypher::NodePatternContext *ctx) { + auto *node = storage_->Create(); + if (ctx->variable()) { + std::string variable = ctx->variable()->accept(this); + node->identifier_ = storage_->Create(variable); + users_identifiers.insert(variable); + } else { + anonymous_identifiers.push_back(&node->identifier_); + } + if (ctx->nodeLabels()) { + node->labels_ = ctx->nodeLabels()->accept(this).as>(); + } + if (ctx->properties()) { + // This can return either properties or parameters + if (ctx->properties()->mapLiteral()) { + node->properties_ = ctx->properties()->accept(this).as>(); + } else { + node->properties_ = ctx->properties()->accept(this).as(); + } + } + return node; +} + +antlrcpp::Any CypherMainVisitor::visitNodeLabels(MemgraphCypher::NodeLabelsContext *ctx) { + std::vector labels; + for (auto *node_label : ctx->nodeLabel()) { + labels.push_back(AddLabel(node_label->accept(this))); + } + return labels; +} + +antlrcpp::Any CypherMainVisitor::visitProperties(MemgraphCypher::PropertiesContext *ctx) { + if (ctx->mapLiteral()) { + return ctx->mapLiteral()->accept(this); + } + // If child is not mapLiteral that means child is params. + MG_ASSERT(ctx->parameter()); + return ctx->parameter()->accept(this); +} + +antlrcpp::Any CypherMainVisitor::visitMapLiteral(MemgraphCypher::MapLiteralContext *ctx) { + std::unordered_map map; + for (int i = 0; i < static_cast(ctx->propertyKeyName().size()); ++i) { + PropertyIx key = ctx->propertyKeyName()[i]->accept(this); + Expression *value = ctx->expression()[i]->accept(this); + if (!map.insert({key, value}).second) { + throw SemanticException("Same key can't appear twice in a map literal."); + } + } + return map; +} + +antlrcpp::Any CypherMainVisitor::visitListLiteral(MemgraphCypher::ListLiteralContext *ctx) { + std::vector expressions; + for (auto expr_ctx_ptr : ctx->expression()) expressions.push_back(expr_ctx_ptr->accept(this)); + return expressions; +} + +antlrcpp::Any CypherMainVisitor::visitPropertyKeyName(MemgraphCypher::PropertyKeyNameContext *ctx) { + return AddProperty(visitChildren(ctx)); +} + +antlrcpp::Any CypherMainVisitor::visitSymbolicName(MemgraphCypher::SymbolicNameContext *ctx) { + if (ctx->EscapedSymbolicName()) { + auto quoted_name = ctx->getText(); + DMG_ASSERT(quoted_name.size() >= 2U && quoted_name[0] == '`' && quoted_name.back() == '`', + "Can't happen. Grammar ensures this"); + // Remove enclosing backticks. + std::string escaped_name = quoted_name.substr(1, static_cast(quoted_name.size()) - 2); + // Unescape remaining backticks. + std::string name; + bool escaped = false; + for (auto c : escaped_name) { + if (escaped) { + if (c == '`') { + name.push_back('`'); + escaped = false; + } else { + DLOG_FATAL("Can't happen. Grammar ensures that."); + } + } else if (c == '`') { + escaped = true; + } else { + name.push_back(c); + } + } + return name; + } + if (ctx->UnescapedSymbolicName()) { + return std::string(ctx->getText()); + } + return ctx->getText(); +} + +antlrcpp::Any CypherMainVisitor::visitPattern(MemgraphCypher::PatternContext *ctx) { + std::vector patterns; + for (auto *pattern_part : ctx->patternPart()) { + patterns.push_back(pattern_part->accept(this)); + } + return patterns; +} + +antlrcpp::Any CypherMainVisitor::visitPatternPart(MemgraphCypher::PatternPartContext *ctx) { + Pattern *pattern = ctx->anonymousPatternPart()->accept(this); + if (ctx->variable()) { + std::string variable = ctx->variable()->accept(this); + pattern->identifier_ = storage_->Create(variable); + users_identifiers.insert(variable); + } else { + anonymous_identifiers.push_back(&pattern->identifier_); + } + return pattern; +} + +antlrcpp::Any CypherMainVisitor::visitPatternElement(MemgraphCypher::PatternElementContext *ctx) { + if (ctx->patternElement()) { + return ctx->patternElement()->accept(this); + } + auto pattern = storage_->Create(); + pattern->atoms_.push_back(ctx->nodePattern()->accept(this).as()); + for (auto *pattern_element_chain : ctx->patternElementChain()) { + std::pair element = pattern_element_chain->accept(this); + pattern->atoms_.push_back(element.first); + pattern->atoms_.push_back(element.second); + } + return pattern; +} + +antlrcpp::Any CypherMainVisitor::visitPatternElementChain(MemgraphCypher::PatternElementChainContext *ctx) { + return std::pair(ctx->relationshipPattern()->accept(this).as(), + ctx->nodePattern()->accept(this).as()); +} + +antlrcpp::Any CypherMainVisitor::visitRelationshipPattern(MemgraphCypher::RelationshipPatternContext *ctx) { + auto *edge = storage_->Create(); + + auto relationshipDetail = ctx->relationshipDetail(); + auto *variableExpansion = relationshipDetail ? relationshipDetail->variableExpansion() : nullptr; + edge->type_ = EdgeAtom::Type::SINGLE; + if (variableExpansion) + std::tie(edge->type_, edge->lower_bound_, edge->upper_bound_) = + variableExpansion->accept(this).as>(); + + if (ctx->leftArrowHead() && !ctx->rightArrowHead()) { + edge->direction_ = EdgeAtom::Direction::IN; + } else if (!ctx->leftArrowHead() && ctx->rightArrowHead()) { + edge->direction_ = EdgeAtom::Direction::OUT; + } else { + // <-[]-> and -[]- is the same thing as far as we understand openCypher + // grammar. + edge->direction_ = EdgeAtom::Direction::BOTH; + } + + if (!relationshipDetail) { + anonymous_identifiers.push_back(&edge->identifier_); + return edge; + } + + if (relationshipDetail->name) { + std::string variable = relationshipDetail->name->accept(this); + edge->identifier_ = storage_->Create(variable); + users_identifiers.insert(variable); + } else { + anonymous_identifiers.push_back(&edge->identifier_); + } + + if (relationshipDetail->relationshipTypes()) { + edge->edge_types_ = ctx->relationshipDetail()->relationshipTypes()->accept(this).as>(); + } + + auto relationshipLambdas = relationshipDetail->relationshipLambda(); + if (variableExpansion) { + if (relationshipDetail->total_weight && edge->type_ != EdgeAtom::Type::WEIGHTED_SHORTEST_PATH) + throw SemanticException( + "Variable for total weight is allowed only with weighted shortest " + "path expansion."); + auto visit_lambda = [this](auto *lambda) { + EdgeAtom::Lambda edge_lambda; + std::string traversed_edge_variable = lambda->traversed_edge->accept(this); + edge_lambda.inner_edge = storage_->Create(traversed_edge_variable); + std::string traversed_node_variable = lambda->traversed_node->accept(this); + edge_lambda.inner_node = storage_->Create(traversed_node_variable); + edge_lambda.expression = lambda->expression()->accept(this); + return edge_lambda; + }; + auto visit_total_weight = [&]() { + if (relationshipDetail->total_weight) { + std::string total_weight_name = relationshipDetail->total_weight->accept(this); + edge->total_weight_ = storage_->Create(total_weight_name); + } else { + anonymous_identifiers.push_back(&edge->total_weight_); + } + }; + switch (relationshipLambdas.size()) { + case 0: + if (edge->type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH) + throw SemanticException( + "Lambda for calculating weights is mandatory with weighted " + "shortest path expansion."); + // In variable expansion inner variables are mandatory. + anonymous_identifiers.push_back(&edge->filter_lambda_.inner_edge); + anonymous_identifiers.push_back(&edge->filter_lambda_.inner_node); + break; + case 1: + if (edge->type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH) { + // For wShortest, the first (and required) lambda is used for weight + // calculation. + edge->weight_lambda_ = visit_lambda(relationshipLambdas[0]); + visit_total_weight(); + // Add mandatory inner variables for filter lambda. + anonymous_identifiers.push_back(&edge->filter_lambda_.inner_edge); + anonymous_identifiers.push_back(&edge->filter_lambda_.inner_node); + } else { + // Other variable expands only have the filter lambda. + edge->filter_lambda_ = visit_lambda(relationshipLambdas[0]); + } + break; + case 2: + if (edge->type_ != EdgeAtom::Type::WEIGHTED_SHORTEST_PATH) + throw SemanticException("Only one filter lambda can be supplied."); + edge->weight_lambda_ = visit_lambda(relationshipLambdas[0]); + visit_total_weight(); + edge->filter_lambda_ = visit_lambda(relationshipLambdas[1]); + break; + default: + throw SemanticException("Only one filter lambda can be supplied."); + } + } else if (!relationshipLambdas.empty()) { + throw SemanticException("Filter lambda is only allowed in variable length expansion."); + } + + auto properties = relationshipDetail->properties(); + switch (properties.size()) { + case 0: + break; + case 1: { + if (properties[0]->mapLiteral()) { + edge->properties_ = properties[0]->accept(this).as>(); + break; + } + MG_ASSERT(properties[0]->parameter()); + edge->properties_ = properties[0]->accept(this).as(); + break; + } + default: + throw SemanticException("Only one property map can be supplied for edge."); + } + + return edge; +} + +antlrcpp::Any CypherMainVisitor::visitRelationshipDetail(MemgraphCypher::RelationshipDetailContext *) { + DLOG_FATAL("Should never be called. See documentation in hpp."); + return 0; +} + +antlrcpp::Any CypherMainVisitor::visitRelationshipLambda(MemgraphCypher::RelationshipLambdaContext *) { + DLOG_FATAL("Should never be called. See documentation in hpp."); + return 0; +} + +antlrcpp::Any CypherMainVisitor::visitRelationshipTypes(MemgraphCypher::RelationshipTypesContext *ctx) { + std::vector types; + for (auto *edge_type : ctx->relTypeName()) { + types.push_back(AddEdgeType(edge_type->accept(this))); + } + return types; +} + +antlrcpp::Any CypherMainVisitor::visitVariableExpansion(MemgraphCypher::VariableExpansionContext *ctx) { + DMG_ASSERT(ctx->expression().size() <= 2U, "Expected 0, 1 or 2 bounds in range literal."); + + EdgeAtom::Type edge_type = EdgeAtom::Type::DEPTH_FIRST; + if (!ctx->getTokens(MemgraphCypher::BFS).empty()) + edge_type = EdgeAtom::Type::BREADTH_FIRST; + else if (!ctx->getTokens(MemgraphCypher::WSHORTEST).empty()) + edge_type = EdgeAtom::Type::WEIGHTED_SHORTEST_PATH; + Expression *lower = nullptr; + Expression *upper = nullptr; + + if (ctx->expression().size() == 0U) { + // Case -[*]- + } else if (ctx->expression().size() == 1U) { + auto dots_tokens = ctx->getTokens(MemgraphCypher::DOTS); + Expression *bound = ctx->expression()[0]->accept(this); + if (!dots_tokens.size()) { + // Case -[*bound]- + if (edge_type != EdgeAtom::Type::WEIGHTED_SHORTEST_PATH) lower = bound; + upper = bound; + } else if (dots_tokens[0]->getSourceInterval().startsAfter(ctx->expression()[0]->getSourceInterval())) { + // Case -[*bound..]- + lower = bound; + } else { + // Case -[*..bound]- + upper = bound; + } + } else { + // Case -[*lbound..rbound]- + lower = ctx->expression()[0]->accept(this); + upper = ctx->expression()[1]->accept(this); + } + if (lower && edge_type == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH) + throw SemanticException("Lower bound is not allowed in weighted shortest path expansion."); + + return std::make_tuple(edge_type, lower, upper); +} + +antlrcpp::Any CypherMainVisitor::visitExpression(MemgraphCypher::ExpressionContext *ctx) { + return static_cast(ctx->expression12()->accept(this)); +} + +// OR. +antlrcpp::Any CypherMainVisitor::visitExpression12(MemgraphCypher::Expression12Context *ctx) { + return LeftAssociativeOperatorExpression(ctx->expression11(), ctx->children, {MemgraphCypher::OR}); +} + +// XOR. +antlrcpp::Any CypherMainVisitor::visitExpression11(MemgraphCypher::Expression11Context *ctx) { + return LeftAssociativeOperatorExpression(ctx->expression10(), ctx->children, {MemgraphCypher::XOR}); +} + +// AND. +antlrcpp::Any CypherMainVisitor::visitExpression10(MemgraphCypher::Expression10Context *ctx) { + return LeftAssociativeOperatorExpression(ctx->expression9(), ctx->children, {MemgraphCypher::AND}); +} + +// NOT. +antlrcpp::Any CypherMainVisitor::visitExpression9(MemgraphCypher::Expression9Context *ctx) { + return PrefixUnaryOperator(ctx->expression8(), ctx->children, {MemgraphCypher::NOT}); +} + +// Comparisons. +// Expresion 1 < 2 < 3 is converted to 1 < 2 && 2 < 3 and then binary operator +// ast node is constructed for each operator. +antlrcpp::Any CypherMainVisitor::visitExpression8(MemgraphCypher::Expression8Context *ctx) { + if (!ctx->partialComparisonExpression().size()) { + // There is no comparison operators. We generate expression7. + return ctx->expression7()->accept(this); + } + + // There is at least one comparison. We need to generate code for each of + // them. We don't call visitPartialComparisonExpression but do everything in + // this function and call expression7-s directly. Since every expression7 + // can be generated twice (because it can appear in two comparisons) code + // generated by whole subtree of expression7 must not have any sideeffects. + // We handle chained comparisons as defined by mathematics, neo4j handles + // them in a very interesting, illogical and incomprehensible way. For + // example in neo4j: + // 1 < 2 < 3 -> true, + // 1 < 2 < 3 < 4 -> false, + // 5 > 3 < 5 > 3 -> true, + // 4 <= 5 < 7 > 6 -> false + // All of those comparisons evaluate to true in memgraph. + std::vector children; + children.push_back(ctx->expression7()->accept(this)); + std::vector operators; + auto partial_comparison_expressions = ctx->partialComparisonExpression(); + for (auto *child : partial_comparison_expressions) { + children.push_back(child->expression7()->accept(this)); + } + // First production is comparison operator. + for (auto *child : partial_comparison_expressions) { + operators.push_back(static_cast(child->children[0])->getSymbol()->getType()); + } + + // Make all comparisons. + Expression *first_operand = children[0]; + std::vector comparisons; + for (int i = 0; i < (int)operators.size(); ++i) { + auto *expr = children[i + 1]; + // TODO: first_operand should only do lookup if it is only calculated and + // not recalculated whole subexpression once again. SymbolGenerator should + // generate symbol for every expresion and then lookup would be possible. + comparisons.push_back(CreateBinaryOperatorByToken(operators[i], first_operand, expr)); + first_operand = expr; + } + + first_operand = comparisons[0]; + // Calculate logical and of results of comparisons. + for (int i = 1; i < (int)comparisons.size(); ++i) { + first_operand = storage_->Create(first_operand, comparisons[i]); + } + return first_operand; +} + +antlrcpp::Any CypherMainVisitor::visitPartialComparisonExpression( + MemgraphCypher::PartialComparisonExpressionContext *) { + DLOG_FATAL("Should never be called. See documentation in hpp."); + return 0; +} + +// Addition and subtraction. +antlrcpp::Any CypherMainVisitor::visitExpression7(MemgraphCypher::Expression7Context *ctx) { + return LeftAssociativeOperatorExpression(ctx->expression6(), ctx->children, + {MemgraphCypher::PLUS, MemgraphCypher::MINUS}); +} + +// Multiplication, division, modding. +antlrcpp::Any CypherMainVisitor::visitExpression6(MemgraphCypher::Expression6Context *ctx) { + return LeftAssociativeOperatorExpression(ctx->expression5(), ctx->children, + {MemgraphCypher::ASTERISK, MemgraphCypher::SLASH, MemgraphCypher::PERCENT}); +} + +// Power. +antlrcpp::Any CypherMainVisitor::visitExpression5(MemgraphCypher::Expression5Context *ctx) { + if (ctx->expression4().size() > 1U) { + // TODO: implement power operator. In neo4j power is left associative and + // int^int -> float. + throw utils::NotYetImplemented("power (^) operator"); + } + return visitChildren(ctx); +} + +// Unary minus and plus. +antlrcpp::Any CypherMainVisitor::visitExpression4(MemgraphCypher::Expression4Context *ctx) { + return PrefixUnaryOperator(ctx->expression3a(), ctx->children, {MemgraphCypher::PLUS, MemgraphCypher::MINUS}); +} + +// IS NULL, IS NOT NULL, STARTS WITH, .. +antlrcpp::Any CypherMainVisitor::visitExpression3a(MemgraphCypher::Expression3aContext *ctx) { + Expression *expression = ctx->expression3b()->accept(this); + + for (auto *op : ctx->stringAndNullOperators()) { + if (op->IS() && op->NOT() && op->CYPHERNULL()) { + expression = + static_cast(storage_->Create(storage_->Create(expression))); + } else if (op->IS() && op->CYPHERNULL()) { + expression = static_cast(storage_->Create(expression)); + } else if (op->IN()) { + expression = + static_cast(storage_->Create(expression, op->expression3b()->accept(this))); + } else if (utils::StartsWith(op->getText(), "=~")) { + auto *regex_match = storage_->Create(); + regex_match->string_expr_ = expression; + regex_match->regex_ = op->expression3b()->accept(this); + expression = regex_match; + } else { + std::string function_name; + if (op->STARTS() && op->WITH()) { + function_name = kStartsWith; + } else if (op->ENDS() && op->WITH()) { + function_name = kEndsWith; + } else if (op->CONTAINS()) { + function_name = kContains; + } else { + throw utils::NotYetImplemented("function '{}'", op->getText()); + } + auto expression2 = op->expression3b()->accept(this); + std::vector args = {expression, expression2}; + expression = static_cast(storage_->Create(function_name, args)); + } + } + return expression; +} +antlrcpp::Any CypherMainVisitor::visitStringAndNullOperators(MemgraphCypher::StringAndNullOperatorsContext *) { + DLOG_FATAL("Should never be called. See documentation in hpp."); + return 0; +} + +antlrcpp::Any CypherMainVisitor::visitExpression3b(MemgraphCypher::Expression3bContext *ctx) { + Expression *expression = ctx->expression2a()->accept(this); + for (auto *list_op : ctx->listIndexingOrSlicing()) { + if (list_op->getTokens(MemgraphCypher::DOTS).size() == 0U) { + // If there is no '..' then we need to create list indexing operator. + expression = storage_->Create(expression, list_op->expression()[0]->accept(this)); + } else if (!list_op->lower_bound && !list_op->upper_bound) { + throw SemanticException("List slicing operator requires at least one bound."); + } else { + Expression *lower_bound_ast = + list_op->lower_bound ? static_cast(list_op->lower_bound->accept(this)) : nullptr; + Expression *upper_bound_ast = + list_op->upper_bound ? static_cast(list_op->upper_bound->accept(this)) : nullptr; + expression = storage_->Create(expression, lower_bound_ast, upper_bound_ast); + } + } + return expression; +} + +antlrcpp::Any CypherMainVisitor::visitListIndexingOrSlicing(MemgraphCypher::ListIndexingOrSlicingContext *) { + DLOG_FATAL("Should never be called. See documentation in hpp."); + return 0; +} + +antlrcpp::Any CypherMainVisitor::visitExpression2a(MemgraphCypher::Expression2aContext *ctx) { + Expression *expression = ctx->expression2b()->accept(this); + if (ctx->nodeLabels()) { + auto labels = ctx->nodeLabels()->accept(this).as>(); + expression = storage_->Create(expression, labels); + } + return expression; +} + +antlrcpp::Any CypherMainVisitor::visitExpression2b(MemgraphCypher::Expression2bContext *ctx) { + Expression *expression = ctx->atom()->accept(this); + for (auto *lookup : ctx->propertyLookup()) { + PropertyIx key = lookup->accept(this); + auto property_lookup = storage_->Create(expression, key); + expression = property_lookup; + } + return expression; +} + +antlrcpp::Any CypherMainVisitor::visitAtom(MemgraphCypher::AtomContext *ctx) { + if (ctx->literal()) { + return ctx->literal()->accept(this); + } else if (ctx->parameter()) { + return static_cast(ctx->parameter()->accept(this).as()); + } else if (ctx->parenthesizedExpression()) { + return static_cast(ctx->parenthesizedExpression()->accept(this)); + } else if (ctx->variable()) { + std::string variable = ctx->variable()->accept(this); + users_identifiers.insert(variable); + return static_cast(storage_->Create(variable)); + } else if (ctx->functionInvocation()) { + return static_cast(ctx->functionInvocation()->accept(this)); + } else if (ctx->COALESCE()) { + std::vector exprs; + for (auto *expr_context : ctx->expression()) { + exprs.emplace_back(expr_context->accept(this).as()); + } + return static_cast(storage_->Create(std::move(exprs))); + } else if (ctx->COUNT()) { + // Here we handle COUNT(*). COUNT(expression) is handled in + // visitFunctionInvocation with other aggregations. This is visible in + // functionInvocation and atom producions in opencypher grammar. + return static_cast(storage_->Create(nullptr, nullptr, Aggregation::Op::COUNT)); + } else if (ctx->ALL()) { + auto *ident = + storage_->Create(ctx->filterExpression()->idInColl()->variable()->accept(this).as()); + Expression *list_expr = ctx->filterExpression()->idInColl()->expression()->accept(this); + if (!ctx->filterExpression()->where()) { + throw SyntaxException("ALL(...) requires a WHERE predicate."); + } + Where *where = ctx->filterExpression()->where()->accept(this); + return static_cast(storage_->Create(ident, list_expr, where)); + } else if (ctx->SINGLE()) { + auto *ident = + storage_->Create(ctx->filterExpression()->idInColl()->variable()->accept(this).as()); + Expression *list_expr = ctx->filterExpression()->idInColl()->expression()->accept(this); + if (!ctx->filterExpression()->where()) { + throw SyntaxException("SINGLE(...) requires a WHERE predicate."); + } + Where *where = ctx->filterExpression()->where()->accept(this); + return static_cast(storage_->Create(ident, list_expr, where)); + } else if (ctx->ANY()) { + auto *ident = + storage_->Create(ctx->filterExpression()->idInColl()->variable()->accept(this).as()); + Expression *list_expr = ctx->filterExpression()->idInColl()->expression()->accept(this); + if (!ctx->filterExpression()->where()) { + throw SyntaxException("ANY(...) requires a WHERE predicate."); + } + Where *where = ctx->filterExpression()->where()->accept(this); + return static_cast(storage_->Create(ident, list_expr, where)); + } else if (ctx->NONE()) { + auto *ident = + storage_->Create(ctx->filterExpression()->idInColl()->variable()->accept(this).as()); + Expression *list_expr = ctx->filterExpression()->idInColl()->expression()->accept(this); + if (!ctx->filterExpression()->where()) { + throw SyntaxException("NONE(...) requires a WHERE predicate."); + } + Where *where = ctx->filterExpression()->where()->accept(this); + return static_cast(storage_->Create(ident, list_expr, where)); + } else if (ctx->REDUCE()) { + auto *accumulator = + storage_->Create(ctx->reduceExpression()->accumulator->accept(this).as()); + Expression *initializer = ctx->reduceExpression()->initial->accept(this); + auto *ident = + storage_->Create(ctx->reduceExpression()->idInColl()->variable()->accept(this).as()); + Expression *list = ctx->reduceExpression()->idInColl()->expression()->accept(this); + Expression *expr = ctx->reduceExpression()->expression().back()->accept(this); + return static_cast(storage_->Create(accumulator, initializer, ident, list, expr)); + } else if (ctx->caseExpression()) { + return static_cast(ctx->caseExpression()->accept(this)); + } else if (ctx->extractExpression()) { + auto *ident = + storage_->Create(ctx->extractExpression()->idInColl()->variable()->accept(this).as()); + Expression *list = ctx->extractExpression()->idInColl()->expression()->accept(this); + Expression *expr = ctx->extractExpression()->expression()->accept(this); + return static_cast(storage_->Create(ident, list, expr)); + } + // TODO: Implement this. We don't support comprehensions, filtering... at + // the moment. + throw utils::NotYetImplemented("atom expression '{}'", ctx->getText()); +} + +antlrcpp::Any CypherMainVisitor::visitParameter(MemgraphCypher::ParameterContext *ctx) { + return storage_->Create(ctx->getStart()->getTokenIndex()); +} + +antlrcpp::Any CypherMainVisitor::visitLiteral(MemgraphCypher::LiteralContext *ctx) { + if (ctx->CYPHERNULL() || ctx->StringLiteral() || ctx->booleanLiteral() || ctx->numberLiteral()) { + int token_position = ctx->getStart()->getTokenIndex(); + if (ctx->CYPHERNULL()) { + return static_cast(storage_->Create(TypedValue(), token_position)); + } else if (context_.is_query_cached) { + // Instead of generating PrimitiveLiteral, we generate a + // ParameterLookup, so that the AST can be cached. This allows for + // varying literals, which are then looked up in the parameters table + // (even though they are not user provided). Note, that NULL always + // generates a PrimitiveLiteral. + return static_cast(storage_->Create(token_position)); + } else if (ctx->StringLiteral()) { + return static_cast(storage_->Create( + visitStringLiteral(ctx->StringLiteral()->getText()).as(), token_position)); + } else if (ctx->booleanLiteral()) { + return static_cast( + storage_->Create(ctx->booleanLiteral()->accept(this).as(), token_position)); + } else if (ctx->numberLiteral()) { + return static_cast( + storage_->Create(ctx->numberLiteral()->accept(this).as(), token_position)); + } + LOG_FATAL("Expected to handle all cases above"); + } else if (ctx->listLiteral()) { + return static_cast( + storage_->Create(ctx->listLiteral()->accept(this).as>())); + } else { + return static_cast(storage_->Create( + ctx->mapLiteral()->accept(this).as>())); + } + return visitChildren(ctx); +} + +antlrcpp::Any CypherMainVisitor::visitParenthesizedExpression(MemgraphCypher::ParenthesizedExpressionContext *ctx) { + return static_cast(ctx->expression()->accept(this)); +} + +antlrcpp::Any CypherMainVisitor::visitNumberLiteral(MemgraphCypher::NumberLiteralContext *ctx) { + if (ctx->integerLiteral()) { + return TypedValue(ctx->integerLiteral()->accept(this).as()); + } else if (ctx->doubleLiteral()) { + return TypedValue(ctx->doubleLiteral()->accept(this).as()); + } else { + // This should never happen, except grammar changes and we don't notice + // change in this production. + DLOG_FATAL("can't happen"); + throw std::exception(); + } +} + +antlrcpp::Any CypherMainVisitor::visitFunctionInvocation(MemgraphCypher::FunctionInvocationContext *ctx) { + if (ctx->DISTINCT()) { + throw utils::NotYetImplemented("DISTINCT function call"); + } + std::string function_name = ctx->functionName()->accept(this); + std::vector expressions; + for (auto *expression : ctx->expression()) { + expressions.push_back(expression->accept(this)); + } + if (expressions.size() == 1U) { + if (function_name == Aggregation::kCount) { + return static_cast(storage_->Create(expressions[0], nullptr, Aggregation::Op::COUNT)); + } + if (function_name == Aggregation::kMin) { + return static_cast(storage_->Create(expressions[0], nullptr, Aggregation::Op::MIN)); + } + if (function_name == Aggregation::kMax) { + return static_cast(storage_->Create(expressions[0], nullptr, Aggregation::Op::MAX)); + } + if (function_name == Aggregation::kSum) { + return static_cast(storage_->Create(expressions[0], nullptr, Aggregation::Op::SUM)); + } + if (function_name == Aggregation::kAvg) { + return static_cast(storage_->Create(expressions[0], nullptr, Aggregation::Op::AVG)); + } + if (function_name == Aggregation::kCollect) { + return static_cast( + storage_->Create(expressions[0], nullptr, Aggregation::Op::COLLECT_LIST)); + } + } + + if (expressions.size() == 2U && function_name == Aggregation::kCollect) { + return static_cast( + storage_->Create(expressions[1], expressions[0], Aggregation::Op::COLLECT_MAP)); + } + + auto is_user_defined_function = [](const std::string &function_name) { + // Dots are present only in user-defined functions, since modules are case-sensitive, so must be user-defined + // functions. Builtin functions should be case insensitive. + return function_name.find('.') != std::string::npos; + }; + + // Don't cache queries which call user-defined functions. User-defined function's return + // types can vary depending on whether the module is reloaded, therefore the cache would + // be invalid. + if (is_user_defined_function(function_name)) { + query_info_.is_cacheable = false; + } + + return static_cast(storage_->Create(function_name, expressions)); +} + +antlrcpp::Any CypherMainVisitor::visitFunctionName(MemgraphCypher::FunctionNameContext *ctx) { + auto function_name = ctx->getText(); + // Dots are present only in user-defined functions, since modules are case-sensitive, so must be user-defined + // functions. Builtin functions should be case insensitive. + if (function_name.find('.') != std::string::npos) { + return function_name; + } + return utils::ToUpperCase(function_name); +} + +antlrcpp::Any CypherMainVisitor::visitDoubleLiteral(MemgraphCypher::DoubleLiteralContext *ctx) { + return ParseDoubleLiteral(ctx->getText()); +} + +antlrcpp::Any CypherMainVisitor::visitIntegerLiteral(MemgraphCypher::IntegerLiteralContext *ctx) { + return ParseIntegerLiteral(ctx->getText()); +} + +antlrcpp::Any CypherMainVisitor::visitStringLiteral(const std::string &escaped) { return ParseStringLiteral(escaped); } + +antlrcpp::Any CypherMainVisitor::visitBooleanLiteral(MemgraphCypher::BooleanLiteralContext *ctx) { + if (ctx->getTokens(MemgraphCypher::TRUE).size()) { + return true; + } + if (ctx->getTokens(MemgraphCypher::FALSE).size()) { + return false; + } + DLOG_FATAL("Shouldn't happend"); + throw std::exception(); +} + +antlrcpp::Any CypherMainVisitor::visitCypherDelete(MemgraphCypher::CypherDeleteContext *ctx) { + auto *del = storage_->Create(); + if (ctx->DETACH()) { + del->detach_ = true; + } + for (auto *expression : ctx->expression()) { + del->expressions_.push_back(expression->accept(this)); + } + return del; +} + +antlrcpp::Any CypherMainVisitor::visitWhere(MemgraphCypher::WhereContext *ctx) { + auto *where = storage_->Create(); + where->expression_ = ctx->expression()->accept(this); + return where; +} + +antlrcpp::Any CypherMainVisitor::visitSet(MemgraphCypher::SetContext *ctx) { + std::vector set_items; + for (auto *set_item : ctx->setItem()) { + set_items.push_back(set_item->accept(this)); + } + return set_items; +} + +antlrcpp::Any CypherMainVisitor::visitSetItem(MemgraphCypher::SetItemContext *ctx) { + // SetProperty + if (ctx->propertyExpression()) { + auto *set_property = storage_->Create(); + set_property->property_lookup_ = ctx->propertyExpression()->accept(this); + set_property->expression_ = ctx->expression()->accept(this); + return static_cast(set_property); + } + + // SetProperties either assignment or update + if (ctx->getTokens(MemgraphCypher::EQ).size() || ctx->getTokens(MemgraphCypher::PLUS_EQ).size()) { + auto *set_properties = storage_->Create(); + set_properties->identifier_ = storage_->Create(ctx->variable()->accept(this).as()); + set_properties->expression_ = ctx->expression()->accept(this); + if (ctx->getTokens(MemgraphCypher::PLUS_EQ).size()) { + set_properties->update_ = true; + } + return static_cast(set_properties); + } + + // SetLabels + auto *set_labels = storage_->Create(); + set_labels->identifier_ = storage_->Create(ctx->variable()->accept(this).as()); + set_labels->labels_ = ctx->nodeLabels()->accept(this).as>(); + return static_cast(set_labels); +} + +antlrcpp::Any CypherMainVisitor::visitRemove(MemgraphCypher::RemoveContext *ctx) { + std::vector remove_items; + for (auto *remove_item : ctx->removeItem()) { + remove_items.push_back(remove_item->accept(this)); + } + return remove_items; +} + +antlrcpp::Any CypherMainVisitor::visitRemoveItem(MemgraphCypher::RemoveItemContext *ctx) { + // RemoveProperty + if (ctx->propertyExpression()) { + auto *remove_property = storage_->Create(); + remove_property->property_lookup_ = ctx->propertyExpression()->accept(this); + return static_cast(remove_property); + } + + // RemoveLabels + auto *remove_labels = storage_->Create(); + remove_labels->identifier_ = storage_->Create(ctx->variable()->accept(this).as()); + remove_labels->labels_ = ctx->nodeLabels()->accept(this).as>(); + return static_cast(remove_labels); +} + +antlrcpp::Any CypherMainVisitor::visitPropertyExpression(MemgraphCypher::PropertyExpressionContext *ctx) { + Expression *expression = ctx->atom()->accept(this); + for (auto *lookup : ctx->propertyLookup()) { + PropertyIx key = lookup->accept(this); + auto property_lookup = storage_->Create(expression, key); + expression = property_lookup; + } + // It is guaranteed by grammar that there is at least one propertyLookup. + return static_cast(expression); +} + +antlrcpp::Any CypherMainVisitor::visitCaseExpression(MemgraphCypher::CaseExpressionContext *ctx) { + Expression *test_expression = ctx->test ? ctx->test->accept(this).as() : nullptr; + auto alternatives = ctx->caseAlternatives(); + // Reverse alternatives so that tree of IfOperators can be built bottom-up. + std::reverse(alternatives.begin(), alternatives.end()); + Expression *else_expression = ctx->else_expression ? ctx->else_expression->accept(this).as() + : storage_->Create(TypedValue()); + for (auto *alternative : alternatives) { + Expression *condition = + test_expression ? storage_->Create(test_expression, alternative->when_expression->accept(this)) + : alternative->when_expression->accept(this).as(); + Expression *then_expression = alternative->then_expression->accept(this); + else_expression = storage_->Create(condition, then_expression, else_expression); + } + return else_expression; +} + +antlrcpp::Any CypherMainVisitor::visitCaseAlternatives(MemgraphCypher::CaseAlternativesContext *) { + DLOG_FATAL("Should never be called. See documentation in hpp."); + return 0; +} + +antlrcpp::Any CypherMainVisitor::visitWith(MemgraphCypher::WithContext *ctx) { + auto *with = storage_->Create(); + in_with_ = true; + with->body_ = ctx->returnBody()->accept(this); + in_with_ = false; + if (ctx->DISTINCT()) { + with->body_.distinct = true; + } + if (ctx->where()) { + with->where_ = ctx->where()->accept(this); + } + return with; +} + +antlrcpp::Any CypherMainVisitor::visitMerge(MemgraphCypher::MergeContext *ctx) { + auto *merge = storage_->Create(); + merge->pattern_ = ctx->patternPart()->accept(this); + for (auto &merge_action : ctx->mergeAction()) { + auto set = merge_action->set()->accept(this).as>(); + if (merge_action->MATCH()) { + merge->on_match_.insert(merge->on_match_.end(), set.begin(), set.end()); + } else { + DMG_ASSERT(merge_action->CREATE(), "Expected ON MATCH or ON CREATE"); + merge->on_create_.insert(merge->on_create_.end(), set.begin(), set.end()); + } + } + return merge; +} + +antlrcpp::Any CypherMainVisitor::visitUnwind(MemgraphCypher::UnwindContext *ctx) { + auto *named_expr = storage_->Create(); + named_expr->expression_ = ctx->expression()->accept(this); + named_expr->name_ = std::string(ctx->variable()->accept(this).as()); + return storage_->Create(named_expr); +} + +antlrcpp::Any CypherMainVisitor::visitFilterExpression(MemgraphCypher::FilterExpressionContext *) { + LOG_FATAL("Should never be called. See documentation in hpp."); + return 0; +} + +antlrcpp::Any CypherMainVisitor::visitForeach(MemgraphCypher::ForeachContext *ctx) { + auto *for_each = storage_->Create(); + + auto *named_expr = storage_->Create(); + named_expr->expression_ = ctx->expression()->accept(this); + named_expr->name_ = std::string(ctx->variable()->accept(this).as()); + for_each->named_expression_ = named_expr; + + for (auto *update_clause_ctx : ctx->updateClause()) { + if (auto *set = update_clause_ctx->set(); set) { + auto set_items = visitSet(set).as>(); + std::copy(set_items.begin(), set_items.end(), std::back_inserter(for_each->clauses_)); + } else if (auto *remove = update_clause_ctx->remove(); remove) { + auto remove_items = visitRemove(remove).as>(); + std::copy(remove_items.begin(), remove_items.end(), std::back_inserter(for_each->clauses_)); + } else if (auto *merge = update_clause_ctx->merge(); merge) { + for_each->clauses_.push_back(visitMerge(merge).as()); + } else if (auto *create = update_clause_ctx->create(); create) { + for_each->clauses_.push_back(visitCreate(create).as()); + } else if (auto *cypher_delete = update_clause_ctx->cypherDelete(); cypher_delete) { + for_each->clauses_.push_back(visitCypherDelete(cypher_delete).as()); + } else { + auto *nested_for_each = update_clause_ctx->foreach (); + MG_ASSERT(nested_for_each != nullptr, "Unexpected clause in FOREACH"); + for_each->clauses_.push_back(visitForeach(nested_for_each).as()); + } + } + + return for_each; +} + +LabelIx CypherMainVisitor::AddLabel(const std::string &name) { return storage_->GetLabelIx(name); } + +PropertyIx CypherMainVisitor::AddProperty(const std::string &name) { return storage_->GetPropertyIx(name); } + +EdgeTypeIx CypherMainVisitor::AddEdgeType(const std::string &name) { return storage_->GetEdgeTypeIx(name); } + +} // namespace memgraph::query::v2::frontend diff --git a/src/query/v2/frontend/ast/cypher_main_visitor.hpp b/src/query/v2/frontend/ast/cypher_main_visitor.hpp new file mode 100644 index 000000000..f0d5ba78b --- /dev/null +++ b/src/query/v2/frontend/ast/cypher_main_visitor.hpp @@ -0,0 +1,886 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +#include + +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/opencypher/generated/MemgraphCypherBaseVisitor.h" +#include "utils/exceptions.hpp" +#include "utils/logging.hpp" + +namespace memgraph::query::v2::frontend { + +using antlropencypher::MemgraphCypher; + +struct ParsingContext { + bool is_query_cached = false; +}; + +class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor { + public: + explicit CypherMainVisitor(ParsingContext context, AstStorage *storage) : context_(context), storage_(storage) {} + + private: + Expression *CreateBinaryOperatorByToken(size_t token, Expression *e1, Expression *e2) { + switch (token) { + case MemgraphCypher::OR: + return storage_->Create(e1, e2); + case MemgraphCypher::XOR: + return storage_->Create(e1, e2); + case MemgraphCypher::AND: + return storage_->Create(e1, e2); + case MemgraphCypher::PLUS: + return storage_->Create(e1, e2); + case MemgraphCypher::MINUS: + return storage_->Create(e1, e2); + case MemgraphCypher::ASTERISK: + return storage_->Create(e1, e2); + case MemgraphCypher::SLASH: + return storage_->Create(e1, e2); + case MemgraphCypher::PERCENT: + return storage_->Create(e1, e2); + case MemgraphCypher::EQ: + return storage_->Create(e1, e2); + case MemgraphCypher::NEQ1: + case MemgraphCypher::NEQ2: + return storage_->Create(e1, e2); + case MemgraphCypher::LT: + return storage_->Create(e1, e2); + case MemgraphCypher::GT: + return storage_->Create(e1, e2); + case MemgraphCypher::LTE: + return storage_->Create(e1, e2); + case MemgraphCypher::GTE: + return storage_->Create(e1, e2); + default: + throw utils::NotYetImplemented("binary operator"); + } + } + + Expression *CreateUnaryOperatorByToken(size_t token, Expression *e) { + switch (token) { + case MemgraphCypher::NOT: + return storage_->Create(e); + case MemgraphCypher::PLUS: + return storage_->Create(e); + case MemgraphCypher::MINUS: + return storage_->Create(e); + default: + throw utils::NotYetImplemented("unary operator"); + } + } + + auto ExtractOperators(std::vector &all_children, + const std::vector &allowed_operators) { + std::vector operators; + for (auto *child : all_children) { + antlr4::tree::TerminalNode *operator_node = nullptr; + if ((operator_node = dynamic_cast(child))) { + if (std::find(allowed_operators.begin(), allowed_operators.end(), operator_node->getSymbol()->getType()) != + allowed_operators.end()) { + operators.push_back(operator_node->getSymbol()->getType()); + } + } + } + return operators; + } + + /** + * Convert opencypher's n-ary production to ast binary operators. + * + * @param _expressions Subexpressions of child for which we construct ast + * operators, for example expression6 if we want to create ast nodes for + * expression7. + */ + template + Expression *LeftAssociativeOperatorExpression(std::vector _expressions, + std::vector all_children, + const std::vector &allowed_operators) { + DMG_ASSERT(_expressions.size(), "can't happen"); + std::vector expressions; + auto operators = ExtractOperators(all_children, allowed_operators); + + for (auto *expression : _expressions) { + expressions.push_back(expression->accept(this)); + } + + Expression *first_operand = expressions[0]; + for (int i = 1; i < (int)expressions.size(); ++i) { + first_operand = CreateBinaryOperatorByToken(operators[i - 1], first_operand, expressions[i]); + } + return first_operand; + } + + template + Expression *PrefixUnaryOperator(TExpression *_expression, std::vector all_children, + const std::vector &allowed_operators) { + DMG_ASSERT(_expression, "can't happen"); + auto operators = ExtractOperators(all_children, allowed_operators); + + Expression *expression = _expression->accept(this); + for (int i = (int)operators.size() - 1; i >= 0; --i) { + expression = CreateUnaryOperatorByToken(operators[i], expression); + } + return expression; + } + + /** + * @return CypherQuery* + */ + antlrcpp::Any visitCypherQuery(MemgraphCypher::CypherQueryContext *ctx) override; + + /** + * @return IndexQuery* + */ + antlrcpp::Any visitIndexQuery(MemgraphCypher::IndexQueryContext *ctx) override; + + /** + * @return ExplainQuery* + */ + antlrcpp::Any visitExplainQuery(MemgraphCypher::ExplainQueryContext *ctx) override; + + /** + * @return ProfileQuery* + */ + antlrcpp::Any visitProfileQuery(MemgraphCypher::ProfileQueryContext *ctx) override; + + /** + * @return InfoQuery* + */ + antlrcpp::Any visitInfoQuery(MemgraphCypher::InfoQueryContext *ctx) override; + + /** + * @return Constraint + */ + antlrcpp::Any visitConstraint(MemgraphCypher::ConstraintContext *ctx) override; + + /** + * @return ConstraintQuery* + */ + antlrcpp::Any visitConstraintQuery(MemgraphCypher::ConstraintQueryContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitAuthQuery(MemgraphCypher::AuthQueryContext *ctx) override; + + /** + * @return DumpQuery* + */ + antlrcpp::Any visitDumpQuery(MemgraphCypher::DumpQueryContext *ctx) override; + + /** + * @return ReplicationQuery* + */ + antlrcpp::Any visitReplicationQuery(MemgraphCypher::ReplicationQueryContext *ctx) override; + + /** + * @return ReplicationQuery* + */ + antlrcpp::Any visitSetReplicationRole(MemgraphCypher::SetReplicationRoleContext *ctx) override; + + /** + * @return ReplicationQuery* + */ + antlrcpp::Any visitShowReplicationRole(MemgraphCypher::ShowReplicationRoleContext *ctx) override; + + /** + * @return ReplicationQuery* + */ + antlrcpp::Any visitRegisterReplica(MemgraphCypher::RegisterReplicaContext *ctx) override; + + /** + * @return ReplicationQuery* + */ + antlrcpp::Any visitDropReplica(MemgraphCypher::DropReplicaContext *ctx) override; + + /** + * @return ReplicationQuery* + */ + antlrcpp::Any visitShowReplicas(MemgraphCypher::ShowReplicasContext *ctx) override; + + /** + * @return LockPathQuery* + */ + antlrcpp::Any visitLockPathQuery(MemgraphCypher::LockPathQueryContext *ctx) override; + + /** + * @return LoadCsvQuery* + */ + antlrcpp::Any visitLoadCsv(MemgraphCypher::LoadCsvContext *ctx) override; + + /** + * @return FreeMemoryQuery* + */ + antlrcpp::Any visitFreeMemoryQuery(MemgraphCypher::FreeMemoryQueryContext *ctx) override; + + /** + * @return TriggerQuery* + */ + antlrcpp::Any visitTriggerQuery(MemgraphCypher::TriggerQueryContext *ctx) override; + + /** + * @return CreateTrigger* + */ + antlrcpp::Any visitCreateTrigger(MemgraphCypher::CreateTriggerContext *ctx) override; + + /** + * @return DropTrigger* + */ + antlrcpp::Any visitDropTrigger(MemgraphCypher::DropTriggerContext *ctx) override; + + /** + * @return ShowTriggers* + */ + antlrcpp::Any visitShowTriggers(MemgraphCypher::ShowTriggersContext *ctx) override; + + /** + * @return IsolationLevelQuery* + */ + antlrcpp::Any visitIsolationLevelQuery(MemgraphCypher::IsolationLevelQueryContext *ctx) override; + + /** + * @return CreateSnapshotQuery* + */ + antlrcpp::Any visitCreateSnapshotQuery(MemgraphCypher::CreateSnapshotQueryContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitStreamQuery(MemgraphCypher::StreamQueryContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitCreateStream(MemgraphCypher::CreateStreamContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitConfigKeyValuePair(MemgraphCypher::ConfigKeyValuePairContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitConfigMap(MemgraphCypher::ConfigMapContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitKafkaCreateStream(MemgraphCypher::KafkaCreateStreamContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitKafkaCreateStreamConfig(MemgraphCypher::KafkaCreateStreamConfigContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitPulsarCreateStreamConfig(MemgraphCypher::PulsarCreateStreamConfigContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitPulsarCreateStream(MemgraphCypher::PulsarCreateStreamContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitCommonCreateStreamConfig(MemgraphCypher::CommonCreateStreamConfigContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitDropStream(MemgraphCypher::DropStreamContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitStartStream(MemgraphCypher::StartStreamContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitStartAllStreams(MemgraphCypher::StartAllStreamsContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitStopStream(MemgraphCypher::StopStreamContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitStopAllStreams(MemgraphCypher::StopAllStreamsContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitShowStreams(MemgraphCypher::ShowStreamsContext *ctx) override; + + /** + * @return StreamQuery* + */ + antlrcpp::Any visitCheckStream(MemgraphCypher::CheckStreamContext *ctx) override; + + /** + * @return SettingQuery* + */ + antlrcpp::Any visitSettingQuery(MemgraphCypher::SettingQueryContext *ctx) override; + + /** + * @return SetSetting* + */ + antlrcpp::Any visitSetSetting(MemgraphCypher::SetSettingContext *ctx) override; + + /** + * @return ShowSetting* + */ + antlrcpp::Any visitShowSetting(MemgraphCypher::ShowSettingContext *ctx) override; + + /** + * @return ShowSettings* + */ + antlrcpp::Any visitShowSettings(MemgraphCypher::ShowSettingsContext *ctx) override; + + /** + * @return VersionQuery* + */ + antlrcpp::Any visitVersionQuery(MemgraphCypher::VersionQueryContext *ctx) override; + + /** + * @return CypherUnion* + */ + antlrcpp::Any visitCypherUnion(MemgraphCypher::CypherUnionContext *ctx) override; + + /** + * @return SingleQuery* + */ + antlrcpp::Any visitSingleQuery(MemgraphCypher::SingleQueryContext *ctx) override; + + /** + * @return Clause* or vector!!! + */ + antlrcpp::Any visitClause(MemgraphCypher::ClauseContext *ctx) override; + + /** + * @return Match* + */ + antlrcpp::Any visitCypherMatch(MemgraphCypher::CypherMatchContext *ctx) override; + + /** + * @return Create* + */ + antlrcpp::Any visitCreate(MemgraphCypher::CreateContext *ctx) override; + + /** + * @return CallProcedure* + */ + antlrcpp::Any visitCallProcedure(MemgraphCypher::CallProcedureContext *ctx) override; + + /** + * @return std::string + */ + antlrcpp::Any visitUserOrRoleName(MemgraphCypher::UserOrRoleNameContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitCreateRole(MemgraphCypher::CreateRoleContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitDropRole(MemgraphCypher::DropRoleContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitShowRoles(MemgraphCypher::ShowRolesContext *ctx) override; + + /** + * @return IndexQuery* + */ + antlrcpp::Any visitCreateIndex(MemgraphCypher::CreateIndexContext *ctx) override; + + /** + * @return DropIndex* + */ + antlrcpp::Any visitDropIndex(MemgraphCypher::DropIndexContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitCreateUser(MemgraphCypher::CreateUserContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitSetPassword(MemgraphCypher::SetPasswordContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitDropUser(MemgraphCypher::DropUserContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitShowUsers(MemgraphCypher::ShowUsersContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitSetRole(MemgraphCypher::SetRoleContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitClearRole(MemgraphCypher::ClearRoleContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitGrantPrivilege(MemgraphCypher::GrantPrivilegeContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitDenyPrivilege(MemgraphCypher::DenyPrivilegeContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitRevokePrivilege(MemgraphCypher::RevokePrivilegeContext *ctx) override; + + /** + * @return AuthQuery::Privilege + */ + antlrcpp::Any visitPrivilege(MemgraphCypher::PrivilegeContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitShowPrivileges(MemgraphCypher::ShowPrivilegesContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitShowRoleForUser(MemgraphCypher::ShowRoleForUserContext *ctx) override; + + /** + * @return AuthQuery* + */ + antlrcpp::Any visitShowUsersForRole(MemgraphCypher::ShowUsersForRoleContext *ctx) override; + + /** + * @return Return* + */ + antlrcpp::Any visitCypherReturn(MemgraphCypher::CypherReturnContext *ctx) override; + + /** + * @return Return* + */ + antlrcpp::Any visitReturnBody(MemgraphCypher::ReturnBodyContext *ctx) override; + + /** + * @return pair> first member is true if + * asterisk was found in return + * expressions. + */ + antlrcpp::Any visitReturnItems(MemgraphCypher::ReturnItemsContext *ctx) override; + + /** + * @return vector + */ + antlrcpp::Any visitReturnItem(MemgraphCypher::ReturnItemContext *ctx) override; + + /** + * @return vector + */ + antlrcpp::Any visitOrder(MemgraphCypher::OrderContext *ctx) override; + + /** + * @return SortItem + */ + antlrcpp::Any visitSortItem(MemgraphCypher::SortItemContext *ctx) override; + + /** + * @return NodeAtom* + */ + antlrcpp::Any visitNodePattern(MemgraphCypher::NodePatternContext *ctx) override; + + /** + * @return vector + */ + antlrcpp::Any visitNodeLabels(MemgraphCypher::NodeLabelsContext *ctx) override; + + /** + * @return unordered_map + */ + antlrcpp::Any visitProperties(MemgraphCypher::PropertiesContext *ctx) override; + + /** + * @return map + */ + antlrcpp::Any visitMapLiteral(MemgraphCypher::MapLiteralContext *ctx) override; + + /** + * @return vector + */ + antlrcpp::Any visitListLiteral(MemgraphCypher::ListLiteralContext *ctx) override; + + /** + * @return PropertyIx + */ + antlrcpp::Any visitPropertyKeyName(MemgraphCypher::PropertyKeyNameContext *ctx) override; + + /** + * @return string + */ + antlrcpp::Any visitSymbolicName(MemgraphCypher::SymbolicNameContext *ctx) override; + + /** + * @return vector + */ + antlrcpp::Any visitPattern(MemgraphCypher::PatternContext *ctx) override; + + /** + * @return Pattern* + */ + antlrcpp::Any visitPatternPart(MemgraphCypher::PatternPartContext *ctx) override; + + /** + * @return Pattern* + */ + antlrcpp::Any visitPatternElement(MemgraphCypher::PatternElementContext *ctx) override; + + /** + * @return vector> + */ + antlrcpp::Any visitPatternElementChain(MemgraphCypher::PatternElementChainContext *ctx) override; + + /** + *@return EdgeAtom* + */ + antlrcpp::Any visitRelationshipPattern(MemgraphCypher::RelationshipPatternContext *ctx) override; + + /** + * This should never be called. Everything is done directly in + * visitRelationshipPattern. + */ + antlrcpp::Any visitRelationshipDetail(MemgraphCypher::RelationshipDetailContext *ctx) override; + + /** + * This should never be called. Everything is done directly in + * visitRelationshipPattern. + */ + antlrcpp::Any visitRelationshipLambda(MemgraphCypher::RelationshipLambdaContext *ctx) override; + + /** + * @return vector + */ + antlrcpp::Any visitRelationshipTypes(MemgraphCypher::RelationshipTypesContext *ctx) override; + + /** + * @return std::tuple. + */ + antlrcpp::Any visitVariableExpansion(MemgraphCypher::VariableExpansionContext *ctx) override; + + /** + * Top level expression, does nothing. + * + * @return Expression* + */ + antlrcpp::Any visitExpression(MemgraphCypher::ExpressionContext *ctx) override; + + /** + * OR. + * + * @return Expression* + */ + antlrcpp::Any visitExpression12(MemgraphCypher::Expression12Context *ctx) override; + + /** + * XOR. + * + * @return Expression* + */ + antlrcpp::Any visitExpression11(MemgraphCypher::Expression11Context *ctx) override; + + /** + * AND. + * + * @return Expression* + */ + antlrcpp::Any visitExpression10(MemgraphCypher::Expression10Context *ctx) override; + + /** + * NOT. + * + * @return Expression* + */ + antlrcpp::Any visitExpression9(MemgraphCypher::Expression9Context *ctx) override; + + /** + * Comparisons. + * + * @return Expression* + */ + antlrcpp::Any visitExpression8(MemgraphCypher::Expression8Context *ctx) override; + + /** + * Never call this. Everything related to generating code for comparison + * operators should be done in visitExpression8. + */ + antlrcpp::Any visitPartialComparisonExpression(MemgraphCypher::PartialComparisonExpressionContext *ctx) override; + + /** + * Addition and subtraction. + * + * @return Expression* + */ + antlrcpp::Any visitExpression7(MemgraphCypher::Expression7Context *ctx) override; + + /** + * Multiplication, division, modding. + * + * @return Expression* + */ + antlrcpp::Any visitExpression6(MemgraphCypher::Expression6Context *ctx) override; + + /** + * Power. + * + * @return Expression* + */ + antlrcpp::Any visitExpression5(MemgraphCypher::Expression5Context *ctx) override; + + /** + * Unary minus and plus. + * + * @return Expression* + */ + antlrcpp::Any visitExpression4(MemgraphCypher::Expression4Context *ctx) override; + + /** + * IS NULL, IS NOT NULL, STARTS WITH, END WITH, =~, ... + * + * @return Expression* + */ + antlrcpp::Any visitExpression3a(MemgraphCypher::Expression3aContext *ctx) override; + + /** + * Does nothing, everything is done in visitExpression3a. + * + * @return Expression* + */ + antlrcpp::Any visitStringAndNullOperators(MemgraphCypher::StringAndNullOperatorsContext *ctx) override; + + /** + * List indexing and slicing. + * + * @return Expression* + */ + antlrcpp::Any visitExpression3b(MemgraphCypher::Expression3bContext *ctx) override; + + /** + * Does nothing, everything is done in visitExpression3b. + */ + antlrcpp::Any visitListIndexingOrSlicing(MemgraphCypher::ListIndexingOrSlicingContext *ctx) override; + + /** + * Node labels test. + * + * @return Expression* + */ + antlrcpp::Any visitExpression2a(MemgraphCypher::Expression2aContext *ctx) override; + + /** + * Property lookup. + * + * @return Expression* + */ + antlrcpp::Any visitExpression2b(MemgraphCypher::Expression2bContext *ctx) override; + + /** + * Literals, params, list comprehension... + * + * @return Expression* + */ + antlrcpp::Any visitAtom(MemgraphCypher::AtomContext *ctx) override; + + /** + * @return ParameterLookup* + */ + antlrcpp::Any visitParameter(MemgraphCypher::ParameterContext *ctx) override; + + /** + * @return Expression* + */ + antlrcpp::Any visitParenthesizedExpression(MemgraphCypher::ParenthesizedExpressionContext *ctx) override; + + /** + * @return Expression* + */ + antlrcpp::Any visitFunctionInvocation(MemgraphCypher::FunctionInvocationContext *ctx) override; + + /** + * @return string - uppercased + */ + antlrcpp::Any visitFunctionName(MemgraphCypher::FunctionNameContext *ctx) override; + + /** + * @return Expression* + */ + antlrcpp::Any visitLiteral(MemgraphCypher::LiteralContext *ctx) override; + + /** + * Convert escaped string from a query to unescaped utf8 string. + * + * @return string + */ + antlrcpp::Any visitStringLiteral(const std::string &escaped); + + /** + * @return bool + */ + antlrcpp::Any visitBooleanLiteral(MemgraphCypher::BooleanLiteralContext *ctx) override; + + /** + * @return TypedValue with either double or int + */ + antlrcpp::Any visitNumberLiteral(MemgraphCypher::NumberLiteralContext *ctx) override; + + /** + * @return int64_t + */ + antlrcpp::Any visitIntegerLiteral(MemgraphCypher::IntegerLiteralContext *ctx) override; + + /** + * @return double + */ + antlrcpp::Any visitDoubleLiteral(MemgraphCypher::DoubleLiteralContext *ctx) override; + + /** + * @return Delete* + */ + antlrcpp::Any visitCypherDelete(MemgraphCypher::CypherDeleteContext *ctx) override; + + /** + * @return Where* + */ + antlrcpp::Any visitWhere(MemgraphCypher::WhereContext *ctx) override; + + /** + * return vector + */ + antlrcpp::Any visitSet(MemgraphCypher::SetContext *ctx) override; + + /** + * @return Clause* + */ + antlrcpp::Any visitSetItem(MemgraphCypher::SetItemContext *ctx) override; + + /** + * return vector + */ + antlrcpp::Any visitRemove(MemgraphCypher::RemoveContext *ctx) override; + + /** + * @return Clause* + */ + antlrcpp::Any visitRemoveItem(MemgraphCypher::RemoveItemContext *ctx) override; + + /** + * @return PropertyLookup* + */ + antlrcpp::Any visitPropertyExpression(MemgraphCypher::PropertyExpressionContext *ctx) override; + + /** + * @return IfOperator* + */ + antlrcpp::Any visitCaseExpression(MemgraphCypher::CaseExpressionContext *ctx) override; + + /** + * Never call this. Ast generation for this production is done in + * @c visitCaseExpression. + */ + antlrcpp::Any visitCaseAlternatives(MemgraphCypher::CaseAlternativesContext *ctx) override; + + /** + * @return With* + */ + antlrcpp::Any visitWith(MemgraphCypher::WithContext *ctx) override; + + /** + * @return Merge* + */ + antlrcpp::Any visitMerge(MemgraphCypher::MergeContext *ctx) override; + + /** + * @return Unwind* + */ + antlrcpp::Any visitUnwind(MemgraphCypher::UnwindContext *ctx) override; + + /** + * Never call this. Ast generation for these expressions should be done by + * explicitly visiting the members of @c FilterExpressionContext. + */ + antlrcpp::Any visitFilterExpression(MemgraphCypher::FilterExpressionContext *) override; + + /** + * @return Foreach* + */ + antlrcpp::Any visitForeach(MemgraphCypher::ForeachContext *ctx) override; + + public: + Query *query() { return query_; } + const static std::string kAnonPrefix; + + struct QueryInfo { + bool is_cacheable{true}; + bool has_load_csv{false}; + }; + + const auto &GetQueryInfo() const { return query_info_; } + + private: + LabelIx AddLabel(const std::string &name); + PropertyIx AddProperty(const std::string &name); + EdgeTypeIx AddEdgeType(const std::string &name); + + ParsingContext context_; + AstStorage *storage_; + + std::unordered_map, + std::unordered_map>> + memory_; + // Set of identifiers from queries. + std::unordered_set users_identifiers; + // Identifiers that user didn't name. + std::vector anonymous_identifiers; + Query *query_ = nullptr; + // All return items which are not variables must be aliased in with. + // We use this variable in visitReturnItem to check if we are in with or + // return. + bool in_with_ = false; + + QueryInfo query_info_; +}; +} // namespace memgraph::query::v2::frontend diff --git a/src/query/v2/frontend/ast/pretty_print.cpp b/src/query/v2/frontend/ast/pretty_print.cpp new file mode 100644 index 000000000..7aaa6ccb1 --- /dev/null +++ b/src/query/v2/frontend/ast/pretty_print.cpp @@ -0,0 +1,311 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/frontend/ast/pretty_print.hpp" + +#include + +#include "query/v2/frontend/ast/ast.hpp" +#include "utils/algorithm.hpp" +#include "utils/string.hpp" + +namespace memgraph::query::v2 { + +namespace { + +class ExpressionPrettyPrinter : public ExpressionVisitor { + public: + explicit ExpressionPrettyPrinter(std::ostream *out); + + // Unary operators + void Visit(NotOperator &op) override; + void Visit(UnaryPlusOperator &op) override; + void Visit(UnaryMinusOperator &op) override; + void Visit(IsNullOperator &op) override; + + // Binary operators + void Visit(OrOperator &op) override; + void Visit(XorOperator &op) override; + void Visit(AndOperator &op) override; + void Visit(AdditionOperator &op) override; + void Visit(SubtractionOperator &op) override; + void Visit(MultiplicationOperator &op) override; + void Visit(DivisionOperator &op) override; + void Visit(ModOperator &op) override; + void Visit(NotEqualOperator &op) override; + void Visit(EqualOperator &op) override; + void Visit(LessOperator &op) override; + void Visit(GreaterOperator &op) override; + void Visit(LessEqualOperator &op) override; + void Visit(GreaterEqualOperator &op) override; + void Visit(InListOperator &op) override; + void Visit(SubscriptOperator &op) override; + + // Other + void Visit(ListSlicingOperator &op) override; + void Visit(IfOperator &op) override; + void Visit(ListLiteral &op) override; + void Visit(MapLiteral &op) override; + void Visit(LabelsTest &op) override; + void Visit(Aggregation &op) override; + void Visit(Function &op) override; + void Visit(Reduce &op) override; + void Visit(Coalesce &op) override; + void Visit(Extract &op) override; + void Visit(All &op) override; + void Visit(Single &op) override; + void Visit(Any &op) override; + void Visit(None &op) override; + void Visit(Identifier &op) override; + void Visit(PrimitiveLiteral &op) override; + void Visit(PropertyLookup &op) override; + void Visit(ParameterLookup &op) override; + void Visit(NamedExpression &op) override; + void Visit(RegexMatch &op) override; + + private: + std::ostream *out_; +}; + +// Declare all of the different `PrintObject` overloads upfront since they're +// mutually recursive. Without this, overload resolution depends on the ordering +// of the overloads within the source, which is quite fragile. + +template +void PrintObject(std::ostream *out, const T &arg); + +void PrintObject(std::ostream *out, const std::string &str); + +void PrintObject(std::ostream *out, Aggregation::Op op); + +void PrintObject(std::ostream *out, Expression *expr); + +void PrintObject(std::ostream *out, Identifier *expr); + +void PrintObject(std::ostream *out, const storage::v3::PropertyValue &value); + +template +void PrintObject(std::ostream *out, const std::vector &vec); + +template +void PrintObject(std::ostream *out, const std::map &map); + +template +void PrintObject(std::ostream *out, const T &arg) { + static_assert(!std::is_convertible::value, + "This overload shouldn't be called with pointers convertible " + "to Expression *. This means your other PrintObject overloads aren't " + "being called for certain AST nodes when they should (or perhaps such " + "overloads don't exist yet)."); + *out << arg; +} + +void PrintObject(std::ostream *out, const std::string &str) { *out << utils::Escape(str); } + +void PrintObject(std::ostream *out, Aggregation::Op op) { *out << Aggregation::OpToString(op); } + +void PrintObject(std::ostream *out, Expression *expr) { + if (expr) { + ExpressionPrettyPrinter printer{out}; + expr->Accept(printer); + } else { + *out << ""; + } +} + +void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast(expr)); } + +void PrintObject(std::ostream *out, const storage::v3::PropertyValue &value) { + switch (value.type()) { + case storage::v3::PropertyValue::Type::Null: + *out << "null"; + break; + + case storage::v3::PropertyValue::Type::String: + PrintObject(out, value.ValueString()); + break; + + case storage::v3::PropertyValue::Type::Bool: + *out << (value.ValueBool() ? "true" : "false"); + break; + + case storage::v3::PropertyValue::Type::Int: + PrintObject(out, value.ValueInt()); + break; + + case storage::v3::PropertyValue::Type::Double: + PrintObject(out, value.ValueDouble()); + break; + + case storage::v3::PropertyValue::Type::List: + PrintObject(out, value.ValueList()); + break; + + case storage::v3::PropertyValue::Type::Map: + PrintObject(out, value.ValueMap()); + break; + case storage::v3::PropertyValue::Type::TemporalData: + PrintObject(out, value.ValueTemporalData()); + break; + } +} + +template +void PrintObject(std::ostream *out, const std::vector &vec) { + *out << "["; + utils::PrintIterable(*out, vec, ", ", [](auto &stream, const auto &item) { PrintObject(&stream, item); }); + *out << "]"; +} + +template +void PrintObject(std::ostream *out, const std::map &map) { + *out << "{"; + utils::PrintIterable(*out, map, ", ", [](auto &stream, const auto &item) { + PrintObject(&stream, item.first); + stream << ": "; + PrintObject(&stream, item.second); + }); + *out << "}"; +} + +template +void PrintOperatorArgs(std::ostream *out, const T &arg) { + *out << " "; + PrintObject(out, arg); + *out << ")"; +} + +template +void PrintOperatorArgs(std::ostream *out, const T &arg, const Ts &...args) { + *out << " "; + PrintObject(out, arg); + PrintOperatorArgs(out, args...); +} + +template +void PrintOperator(std::ostream *out, const std::string &name, const Ts &...args) { + *out << "(" << name; + PrintOperatorArgs(out, args...); +} + +ExpressionPrettyPrinter::ExpressionPrettyPrinter(std::ostream *out) : out_(out) {} + +#define UNARY_OPERATOR_VISIT(OP_NODE, OP_STR) \ + void ExpressionPrettyPrinter::Visit(OP_NODE &op) { PrintOperator(out_, OP_STR, op.expression_); } + +UNARY_OPERATOR_VISIT(NotOperator, "Not"); +UNARY_OPERATOR_VISIT(UnaryPlusOperator, "+"); +UNARY_OPERATOR_VISIT(UnaryMinusOperator, "-"); +UNARY_OPERATOR_VISIT(IsNullOperator, "IsNull"); + +#undef UNARY_OPERATOR_VISIT + +#define BINARY_OPERATOR_VISIT(OP_NODE, OP_STR) \ + void ExpressionPrettyPrinter::Visit(OP_NODE &op) { PrintOperator(out_, OP_STR, op.expression1_, op.expression2_); } + +BINARY_OPERATOR_VISIT(OrOperator, "Or"); +BINARY_OPERATOR_VISIT(XorOperator, "Xor"); +BINARY_OPERATOR_VISIT(AndOperator, "And"); +BINARY_OPERATOR_VISIT(AdditionOperator, "+"); +BINARY_OPERATOR_VISIT(SubtractionOperator, "-"); +BINARY_OPERATOR_VISIT(MultiplicationOperator, "*"); +BINARY_OPERATOR_VISIT(DivisionOperator, "/"); +BINARY_OPERATOR_VISIT(ModOperator, "%"); +BINARY_OPERATOR_VISIT(NotEqualOperator, "!="); +BINARY_OPERATOR_VISIT(EqualOperator, "=="); +BINARY_OPERATOR_VISIT(LessOperator, "<"); +BINARY_OPERATOR_VISIT(GreaterOperator, ">"); +BINARY_OPERATOR_VISIT(LessEqualOperator, "<="); +BINARY_OPERATOR_VISIT(GreaterEqualOperator, ">="); +BINARY_OPERATOR_VISIT(InListOperator, "In"); +BINARY_OPERATOR_VISIT(SubscriptOperator, "Subscript"); + +#undef BINARY_OPERATOR_VISIT + +void ExpressionPrettyPrinter::Visit(ListSlicingOperator &op) { + PrintOperator(out_, "ListSlicing", op.list_, op.lower_bound_, op.upper_bound_); +} + +void ExpressionPrettyPrinter::Visit(IfOperator &op) { + PrintOperator(out_, "If", op.condition_, op.then_expression_, op.else_expression_); +} + +void ExpressionPrettyPrinter::Visit(ListLiteral &op) { PrintOperator(out_, "ListLiteral", op.elements_); } + +void ExpressionPrettyPrinter::Visit(MapLiteral &op) { + std::map map; + for (const auto &kv : op.elements_) { + map[kv.first.name] = kv.second; + } + PrintObject(out_, map); +} + +void ExpressionPrettyPrinter::Visit(LabelsTest &op) { PrintOperator(out_, "LabelsTest", op.expression_); } + +void ExpressionPrettyPrinter::Visit(Aggregation &op) { PrintOperator(out_, "Aggregation", op.op_); } + +void ExpressionPrettyPrinter::Visit(Function &op) { PrintOperator(out_, "Function", op.function_name_, op.arguments_); } + +void ExpressionPrettyPrinter::Visit(Reduce &op) { + PrintOperator(out_, "Reduce", op.accumulator_, op.initializer_, op.identifier_, op.list_, op.expression_); +} + +void ExpressionPrettyPrinter::Visit(Coalesce &op) { PrintOperator(out_, "Coalesce", op.expressions_); } + +void ExpressionPrettyPrinter::Visit(Extract &op) { + PrintOperator(out_, "Extract", op.identifier_, op.list_, op.expression_); +} + +void ExpressionPrettyPrinter::Visit(All &op) { + PrintOperator(out_, "All", op.identifier_, op.list_expression_, op.where_->expression_); +} + +void ExpressionPrettyPrinter::Visit(Single &op) { + PrintOperator(out_, "Single", op.identifier_, op.list_expression_, op.where_->expression_); +} + +void ExpressionPrettyPrinter::Visit(Any &op) { + PrintOperator(out_, "Any", op.identifier_, op.list_expression_, op.where_->expression_); +} + +void ExpressionPrettyPrinter::Visit(None &op) { + PrintOperator(out_, "None", op.identifier_, op.list_expression_, op.where_->expression_); +} + +void ExpressionPrettyPrinter::Visit(Identifier &op) { PrintOperator(out_, "Identifier", op.name_); } + +void ExpressionPrettyPrinter::Visit(PrimitiveLiteral &op) { PrintObject(out_, op.value_); } + +void ExpressionPrettyPrinter::Visit(PropertyLookup &op) { + PrintOperator(out_, "PropertyLookup", op.expression_, op.property_.name); +} + +void ExpressionPrettyPrinter::Visit(ParameterLookup &op) { PrintOperator(out_, "ParameterLookup", op.token_position_); } + +void ExpressionPrettyPrinter::Visit(NamedExpression &op) { + PrintOperator(out_, "NamedExpression", op.name_, op.expression_); +} + +void ExpressionPrettyPrinter::Visit(RegexMatch &op) { PrintOperator(out_, "=~", op.string_expr_, op.regex_); } + +} // namespace + +void PrintExpression(Expression *expr, std::ostream *out) { + ExpressionPrettyPrinter printer{out}; + expr->Accept(printer); +} + +void PrintExpression(NamedExpression *expr, std::ostream *out) { + ExpressionPrettyPrinter printer{out}; + expr->Accept(printer); +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/ast/pretty_print.hpp b/src/query/v2/frontend/ast/pretty_print.hpp new file mode 100644 index 000000000..d6047c349 --- /dev/null +++ b/src/query/v2/frontend/ast/pretty_print.hpp @@ -0,0 +1,23 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "query/v2/frontend/ast/ast.hpp" + +namespace memgraph::query::v2 { + +void PrintExpression(Expression *expr, std::ostream *out); +void PrintExpression(NamedExpression *expr, std::ostream *out); + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/opencypher/grammar/Cypher.g4 b/src/query/v2/frontend/opencypher/grammar/Cypher.g4 new file mode 100644 index 000000000..6ce84db1a --- /dev/null +++ b/src/query/v2/frontend/opencypher/grammar/Cypher.g4 @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2015-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +parser grammar Cypher; + +options { tokenVocab=CypherLexer; } + +cypher : statement ';'? EOF ; + +statement : query ; + +query : cypherQuery + | indexQuery + | explainQuery + | profileQuery + | infoQuery + | constraintQuery + ; + +constraintQuery : ( CREATE | DROP ) CONSTRAINT ON constraint ; + +constraint : '(' nodeName=variable ':' labelName ')' ASSERT EXISTS '(' constraintPropertyList ')' + | '(' nodeName=variable ':' labelName ')' ASSERT constraintPropertyList IS UNIQUE + | '(' nodeName=variable ':' labelName ')' ASSERT '(' constraintPropertyList ')' IS NODE KEY + ; + +constraintPropertyList : variable propertyLookup ( ',' variable propertyLookup )* ; + +storageInfo : STORAGE INFO ; + +indexInfo : INDEX INFO ; + +constraintInfo : CONSTRAINT INFO ; + +infoQuery : SHOW ( storageInfo | indexInfo | constraintInfo ) ; + +explainQuery : EXPLAIN cypherQuery ; + +profileQuery : PROFILE cypherQuery ; + +cypherQuery : singleQuery ( cypherUnion )* ( queryMemoryLimit )? ; + +indexQuery : createIndex | dropIndex; + +singleQuery : clause ( clause )* ; + +cypherUnion : ( UNION ALL singleQuery ) + | ( UNION singleQuery ) + ; + +clause : cypherMatch + | unwind + | merge + | create + | set + | cypherDelete + | remove + | with + | cypherReturn + | callProcedure + ; + +cypherMatch : OPTIONAL? MATCH pattern where? ; + +unwind : UNWIND expression AS variable ; + +merge : MERGE patternPart ( mergeAction )* ; + +mergeAction : ( ON MATCH set ) + | ( ON CREATE set ) + ; + +create : CREATE pattern ; + +set : SET setItem ( ',' setItem )* ; + +setItem : ( propertyExpression '=' expression ) + | ( variable '=' expression ) + | ( variable '+=' expression ) + | ( variable nodeLabels ) + ; + +cypherDelete : DETACH? DELETE expression ( ',' expression )* ; + +remove : REMOVE removeItem ( ',' removeItem )* ; + +removeItem : ( variable nodeLabels ) + | propertyExpression + ; + +with : WITH ( DISTINCT )? returnBody ( where )? ; + +cypherReturn : RETURN ( DISTINCT )? returnBody ; + +callProcedure : CALL procedureName '(' ( expression ( ',' expression )* )? ')' ( procedureMemoryLimit )? ( yieldProcedureResults )? ; + +procedureName : symbolicName ( '.' symbolicName )* ; + +yieldProcedureResults : YIELD ( '*' | ( procedureResult ( ',' procedureResult )* ) ) ; + +memoryLimit : MEMORY ( UNLIMITED | LIMIT literal ( MB | KB ) ) ; + +queryMemoryLimit : QUERY memoryLimit ; + +procedureMemoryLimit : PROCEDURE memoryLimit ; + +procedureResult : ( variable AS variable ) | variable ; + +returnBody : returnItems ( order )? ( skip )? ( limit )? ; + +returnItems : ( '*' ( ',' returnItem )* ) + | ( returnItem ( ',' returnItem )* ) + ; + +returnItem : ( expression AS variable ) + | expression + ; + +order : ORDER BY sortItem ( ',' sortItem )* ; + +skip : L_SKIP expression ; + +limit : LIMIT expression ; + +sortItem : expression ( ASCENDING | ASC | DESCENDING | DESC )? ; + +where : WHERE expression ; + +pattern : patternPart ( ',' patternPart )* ; + +patternPart : ( variable '=' anonymousPatternPart ) + | anonymousPatternPart + ; + +anonymousPatternPart : patternElement ; + +patternElement : ( nodePattern ( patternElementChain )* ) + | ( '(' patternElement ')' ) + ; + +nodePattern : '(' ( variable )? ( nodeLabels )? ( properties )? ')' ; + +patternElementChain : relationshipPattern nodePattern ; + +relationshipPattern : ( leftArrowHead dash ( relationshipDetail )? dash rightArrowHead ) + | ( leftArrowHead dash ( relationshipDetail )? dash ) + | ( dash ( relationshipDetail )? dash rightArrowHead ) + | ( dash ( relationshipDetail )? dash ) + ; + +leftArrowHead : '<' | LeftArrowHeadPart ; +rightArrowHead : '>' | RightArrowHeadPart ; +dash : '-' | DashPart ; + +relationshipDetail : '[' ( name=variable )? ( relationshipTypes )? ( variableExpansion )? properties ']' + | '[' ( name=variable )? ( relationshipTypes )? ( variableExpansion )? relationshipLambda ( total_weight=variable )? (relationshipLambda )? ']' + | '[' ( name=variable )? ( relationshipTypes )? ( variableExpansion )? (properties )* ( relationshipLambda total_weight=variable )? (relationshipLambda )? ']'; + +relationshipLambda: '(' traversed_edge=variable ',' traversed_node=variable '|' expression ')'; + +variableExpansion : '*' (BFS | WSHORTEST)? ( expression )? ( '..' ( expression )? )? ; + +properties : mapLiteral + | parameter + ; + +relationshipTypes : ':' relTypeName ( '|' ':'? relTypeName )* ; + +nodeLabels : nodeLabel ( nodeLabel )* ; + +nodeLabel : ':' labelName ; + +labelName : symbolicName ; + +relTypeName : symbolicName ; + +expression : expression12 ; + +expression12 : expression11 ( OR expression11 )* ; + +expression11 : expression10 ( XOR expression10 )* ; + +expression10 : expression9 ( AND expression9 )* ; + +expression9 : ( NOT )* expression8 ; + +expression8 : expression7 ( partialComparisonExpression )* ; + +expression7 : expression6 ( ( '+' expression6 ) | ( '-' expression6 ) )* ; + +expression6 : expression5 ( ( '*' expression5 ) | ( '/' expression5 ) | ( '%' expression5 ) )* ; + +expression5 : expression4 ( '^' expression4 )* ; + +expression4 : ( ( '+' | '-' ) )* expression3a ; + +expression3a : expression3b ( stringAndNullOperators )* ; + +stringAndNullOperators : ( ( ( ( '=~' ) | ( IN ) | ( STARTS WITH ) | ( ENDS WITH ) | ( CONTAINS ) ) expression3b) | ( IS CYPHERNULL ) | ( IS NOT CYPHERNULL ) ) ; + +expression3b : expression2a ( listIndexingOrSlicing )* ; + +listIndexingOrSlicing : ( '[' expression ']' ) + | ( '[' lower_bound=expression? '..' upper_bound=expression? ']' ) + ; + +expression2a : expression2b ( nodeLabels )? ; + +expression2b : atom ( propertyLookup )* ; + +atom : literal + | parameter + | caseExpression + | ( COUNT '(' '*' ')' ) + | listComprehension + | patternComprehension + | ( FILTER '(' filterExpression ')' ) + | ( EXTRACT '(' extractExpression ')' ) + | ( REDUCE '(' reduceExpression ')' ) + | ( COALESCE '(' expression ( ',' expression )* ')' ) + | ( ALL '(' filterExpression ')' ) + | ( ANY '(' filterExpression ')' ) + | ( NONE '(' filterExpression ')' ) + | ( SINGLE '(' filterExpression ')' ) + | relationshipsPattern + | parenthesizedExpression + | functionInvocation + | variable + ; + +literal : numberLiteral + | StringLiteral + | booleanLiteral + | CYPHERNULL + | mapLiteral + | listLiteral + ; + +booleanLiteral : TRUE + | FALSE + ; + +listLiteral : '[' ( expression ( ',' expression )* )? ']' ; + +partialComparisonExpression : ( '=' expression7 ) + | ( '<>' expression7 ) + | ( '!=' expression7 ) + | ( '<' expression7 ) + | ( '>' expression7 ) + | ( '<=' expression7 ) + | ( '>=' expression7 ) + ; + +parenthesizedExpression : '(' expression ')' ; + +relationshipsPattern : nodePattern ( patternElementChain )+ ; + +filterExpression : idInColl ( where )? ; + +reduceExpression : accumulator=variable '=' initial=expression ',' idInColl '|' expression ; + +extractExpression : idInColl '|' expression ; + +idInColl : variable IN expression ; + +functionInvocation : functionName '(' ( DISTINCT )? ( expression ( ',' expression )* )? ')' ; + +functionName : symbolicName ( '.' symbolicName )* ; + +listComprehension : '[' filterExpression ( '|' expression )? ']' ; + +patternComprehension : '[' ( variable '=' )? relationshipsPattern ( WHERE expression )? '|' expression ']' ; + +propertyLookup : '.' ( propertyKeyName ) ; + +caseExpression : ( ( CASE ( caseAlternatives )+ ) | ( CASE test=expression ( caseAlternatives )+ ) ) ( ELSE else_expression=expression )? END ; + +caseAlternatives : WHEN when_expression=expression THEN then_expression=expression ; + +variable : symbolicName ; + +numberLiteral : doubleLiteral + | integerLiteral + ; + +mapLiteral : '{' ( propertyKeyName ':' expression ( ',' propertyKeyName ':' expression )* )? '}' ; + +parameter : '$' ( symbolicName | DecimalLiteral ) ; + +propertyExpression : atom ( propertyLookup )+ ; + +propertyKeyName : symbolicName ; + +integerLiteral : DecimalLiteral + | OctalLiteral + | HexadecimalLiteral + ; + +createIndex : CREATE INDEX ON ':' labelName ( '(' propertyKeyName ')' )? ; + +dropIndex : DROP INDEX ON ':' labelName ( '(' propertyKeyName ')' )? ; + +doubleLiteral : FloatingLiteral ; + +cypherKeyword : ALL + | AND + | ANY + | AS + | ASC + | ASCENDING + | ASSERT + | BFS + | BY + | CALL + | CASE + | CONSTRAINT + | CONTAINS + | COUNT + | CREATE + | CYPHERNULL + | DELETE + | DESC + | DESCENDING + | DETACH + | DISTINCT + | ELSE + | END + | ENDS + | EXISTS + | EXPLAIN + | EXTRACT + | FALSE + | FILTER + | IN + | INDEX + | INFO + | IS + | KEY + | LIMIT + | L_SKIP + | MATCH + | MERGE + | NODE + | NONE + | NOT + | ON + | OPTIONAL + | OR + | ORDER + | PROCEDURE + | PROFILE + | QUERY + | REDUCE + | REMOVE + | RETURN + | SET + | SHOW + | SINGLE + | STARTS + | STORAGE + | THEN + | TRUE + | UNION + | UNIQUE + | UNWIND + | WHEN + | WHERE + | WITH + | WSHORTEST + | XOR + | YIELD + ; + +symbolicName : UnescapedSymbolicName + | EscapedSymbolicName + | cypherKeyword + ; diff --git a/src/query/v2/frontend/opencypher/grammar/CypherLexer.g4 b/src/query/v2/frontend/opencypher/grammar/CypherLexer.g4 new file mode 100644 index 000000000..1377fbc82 --- /dev/null +++ b/src/query/v2/frontend/opencypher/grammar/CypherLexer.g4 @@ -0,0 +1,208 @@ +/* + * When changing this grammar make sure to update constants in + * src/query/frontend/stripped_lexer_constants.hpp (kKeywords, kSpecialTokens + * and bitsets) if needed. + */ + +lexer grammar CypherLexer ; + +import UnicodeCategories ; + +/* Skip whitespace and comments. */ +Skipped : ( Whitespace | Comment ) -> skip ; + +fragment Whitespace : '\u0020' + | [\u0009-\u000D] + | [\u001C-\u001F] + | '\u1680' | '\u180E' + | [\u2000-\u200A] + | '\u2028' | '\u2029' + | '\u205F' + | '\u3000' + | '\u00A0' + | '\u202F' + ; + +fragment Comment : '/*' .*? '*/' + | '//' ~[\r\n]* + ; + +/* Special symbols. */ +LPAREN : '(' ; +RPAREN : ')' ; +LBRACK : '[' ; +RBRACK : ']' ; +LBRACE : '{' ; +RBRACE : '}' ; + +COMMA : ',' ; +DOT : '.' ; +DOTS : '..' ; +COLON : ':' ; +SEMICOLON : ';' ; +DOLLAR : '$' ; +PIPE : '|' ; + +EQ : '=' ; +LT : '<' ; +GT : '>' ; +LTE : '<=' ; +GTE : '>=' ; +NEQ1 : '<>' ; +NEQ2 : '!=' ; +SIM : '=~' ; + +PLUS : '+' ; +MINUS : '-' ; +ASTERISK : '*' ; +SLASH : '/' ; +PERCENT : '%' ; +CARET : '^' ; +PLUS_EQ : '+=' ; + +/* Some random unicode characters that can be used to draw arrows. */ +LeftArrowHeadPart : '⟨' | '〈' | '﹤' | '＜' ; +RightArrowHeadPart : '⟩' | '〉' | '﹥' | '＞' ; +DashPart : '­' | '‐' | '‑' | '‒' | '–' | '—' | '―' + | '−' | '﹘' | '﹣' | '－' + ; + +/* Cypher reserved words. */ +ALL : A L L ; +AND : A N D ; +ANY : A N Y ; +AS : A S ; +ASC : A S C ; +ASCENDING : A S C E N D I N G ; +ASSERT : A S S E R T ; +BFS : B F S ; +BY : B Y ; +CALL : C A L L ; +CASE : C A S E ; +COALESCE : C O A L E S C E ; +CONSTRAINT : C O N S T R A I N T ; +CONTAINS : C O N T A I N S ; +COUNT : C O U N T ; +CREATE : C R E A T E ; +CYPHERNULL : N U L L ; +DELETE : D E L E T E ; +DESC : D E S C ; +DESCENDING : D E S C E N D I N G ; +DETACH : D E T A C H ; +DISTINCT : D I S T I N C T ; +DROP : D R O P ; +ELSE : E L S E ; +END : E N D ; +ENDS : E N D S ; +EXISTS : E X I S T S ; +EXPLAIN : E X P L A I N ; +EXTRACT : E X T R A C T ; +FALSE : F A L S E ; +FILTER : F I L T E R ; +IN : I N ; +INDEX : I N D E X ; +INFO : I N F O ; +IS : I S ; +KB : K B ; +KEY : K E Y ; +LIMIT : L I M I T ; +L_SKIP : S K I P ; +MATCH : M A T C H ; +MB : M B ; +MEMORY : M E M O R Y ; +MERGE : M E R G E ; +NODE : N O D E ; +NONE : N O N E ; +NOT : N O T ; +ON : O N ; +OPTIONAL : O P T I O N A L ; +OR : O R ; +ORDER : O R D E R ; +PROCEDURE : P R O C E D U R E ; +PROFILE : P R O F I L E ; +QUERY : Q U E R Y ; +REDUCE : R E D U C E ; +REMOVE : R E M O V E ; +RETURN : R E T U R N ; +SET : S E T ; +SHOW : S H O W ; +SINGLE : S I N G L E ; +STARTS : S T A R T S ; +STORAGE : S T O R A G E ; +THEN : T H E N ; +TRUE : T R U E ; +UNION : U N I O N ; +UNIQUE : U N I Q U E ; +UNLIMITED : U N L I M I T E D ; +UNWIND : U N W I N D ; +WHEN : W H E N ; +WHERE : W H E R E ; +WITH : W I T H ; +WSHORTEST : W S H O R T E S T ; +XOR : X O R ; +YIELD : Y I E L D ; + +/* Double and single quoted string literals. */ +StringLiteral : '"' ( ~[\\"] | EscapeSequence )* '"' + | '\'' ( ~[\\'] | EscapeSequence )* '\'' + ; + +fragment EscapeSequence : '\\' ( B | F | N | R | T | '\\' | '\'' | '"' ) + | '\\u' HexDigit HexDigit HexDigit HexDigit + | '\\U' HexDigit HexDigit HexDigit HexDigit + HexDigit HexDigit HexDigit HexDigit + ; + +/* Number literals. */ +DecimalLiteral : '0' | NonZeroDigit ( DecDigit )* ; +OctalLiteral : '0' ( OctDigit )+ ; +HexadecimalLiteral : '0x' ( HexDigit )+ ; +FloatingLiteral : DecDigit* '.' DecDigit+ ( E '-'? DecDigit+ )? + | DecDigit+ ( '.' DecDigit* )? ( E '-'? DecDigit+ ) + | DecDigit+ ( E '-'? DecDigit+ ) + ; + +fragment NonZeroDigit : [1-9] ; +fragment DecDigit : [0-9] ; +fragment OctDigit : [0-7] ; +fragment HexDigit : [0-9] | [a-f] | [A-F] ; + +/* Symbolic names. */ +UnescapedSymbolicName : IdentifierStart ( IdentifierPart )* ; +EscapedSymbolicName : ( '`' ~[`]* '`' )+ ; + +/** + * Based on the unicode identifier and pattern syntax + * (http://www.unicode.org/reports/tr31/) + * and extended with a few characters. + */ +IdentifierStart : ID_Start | Pc ; +IdentifierPart : ID_Continue | Sc ; + +/* Hack for case-insensitive reserved words */ +fragment A : 'A' | 'a' ; +fragment B : 'B' | 'b' ; +fragment C : 'C' | 'c' ; +fragment D : 'D' | 'd' ; +fragment E : 'E' | 'e' ; +fragment F : 'F' | 'f' ; +fragment G : 'G' | 'g' ; +fragment H : 'H' | 'h' ; +fragment I : 'I' | 'i' ; +fragment J : 'J' | 'j' ; +fragment K : 'K' | 'k' ; +fragment L : 'L' | 'l' ; +fragment M : 'M' | 'm' ; +fragment N : 'N' | 'n' ; +fragment O : 'O' | 'o' ; +fragment P : 'P' | 'p' ; +fragment Q : 'Q' | 'q' ; +fragment R : 'R' | 'r' ; +fragment S : 'S' | 's' ; +fragment T : 'T' | 't' ; +fragment U : 'U' | 'u' ; +fragment V : 'V' | 'v' ; +fragment W : 'W' | 'w' ; +fragment X : 'X' | 'x' ; +fragment Y : 'Y' | 'y' ; +fragment Z : 'Z' | 'z' ; diff --git a/src/query/v2/frontend/opencypher/grammar/MemgraphCypher.g4 b/src/query/v2/frontend/opencypher/grammar/MemgraphCypher.g4 new file mode 100644 index 000000000..b412a474a --- /dev/null +++ b/src/query/v2/frontend/opencypher/grammar/MemgraphCypher.g4 @@ -0,0 +1,376 @@ +/* + * Copyright 2021 Memgraph Ltd. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source + * License, and you may not use this file except in compliance with the Business Source License. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +/* Memgraph specific part of Cypher grammar with enterprise features. */ + +parser grammar MemgraphCypher ; + +options { tokenVocab=MemgraphCypherLexer; } + +import Cypher ; + +memgraphCypherKeyword : cypherKeyword + | AFTER + | ALTER + | ASYNC + | AUTH + | BAD + | BATCH_INTERVAL + | BATCH_LIMIT + | BATCH_SIZE + | BEFORE + | BOOTSTRAP_SERVERS + | CHECK + | CLEAR + | COMMIT + | COMMITTED + | CONFIG + | CONFIGS + | CONSUMER_GROUP + | CREDENTIALS + | CSV + | DATA + | DELIMITER + | DATABASE + | DENY + | DROP + | DUMP + | EXECUTE + | FOR + | FOREACH + | FREE + | FROM + | GLOBAL + | GRANT + | HEADER + | IDENTIFIED + | ISOLATION + | KAFKA + | LEVEL + | LOAD + | LOCK + | MAIN + | MODE + | NEXT + | NO + | PASSWORD + | PULSAR + | PORT + | PRIVILEGES + | READ + | REGISTER + | REPLICA + | REPLICAS + | REPLICATION + | REVOKE + | ROLE + | ROLES + | QUOTE + | SESSION + | SETTING + | SETTINGS + | SNAPSHOT + | START + | STATS + | STREAM + | STREAMS + | SYNC + | TIMEOUT + | TO + | TOPICS + | TRANSACTION + | TRANSFORM + | TRIGGER + | TRIGGERS + | UNCOMMITTED + | UNLOCK + | UPDATE + | USER + | USERS + | VERSION + ; + +symbolicName : UnescapedSymbolicName + | EscapedSymbolicName + | memgraphCypherKeyword + ; + +query : cypherQuery + | indexQuery + | explainQuery + | profileQuery + | infoQuery + | constraintQuery + | authQuery + | dumpQuery + | replicationQuery + | lockPathQuery + | freeMemoryQuery + | triggerQuery + | isolationLevelQuery + | createSnapshotQuery + | streamQuery + | settingQuery + | versionQuery + ; + +authQuery : createRole + | dropRole + | showRoles + | createUser + | setPassword + | dropUser + | showUsers + | setRole + | clearRole + | grantPrivilege + | denyPrivilege + | revokePrivilege + | showPrivileges + | showRoleForUser + | showUsersForRole + ; + +replicationQuery : setReplicationRole + | showReplicationRole + | registerReplica + | dropReplica + | showReplicas + ; + +triggerQuery : createTrigger + | dropTrigger + | showTriggers + ; + +clause : cypherMatch + | unwind + | merge + | create + | set + | cypherDelete + | remove + | with + | cypherReturn + | callProcedure + | loadCsv + | foreach + ; + +updateClause : set + | remove + | create + | merge + | cypherDelete + | foreach + ; + +foreach : FOREACH '(' variable IN expression '|' updateClause+ ')' ; + +streamQuery : checkStream + | createStream + | dropStream + | startStream + | startAllStreams + | stopStream + | stopAllStreams + | showStreams + ; + +settingQuery : setSetting + | showSetting + | showSettings + ; + +loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER + ( IGNORE BAD ) ? + ( DELIMITER delimiter ) ? + ( QUOTE quote ) ? + AS rowVar ; + +csvFile : literal ; + +delimiter : literal ; + +quote : literal ; + +rowVar : variable ; + +userOrRoleName : symbolicName ; + +createRole : CREATE ROLE role=userOrRoleName ; + +dropRole : DROP ROLE role=userOrRoleName ; + +showRoles : SHOW ROLES ; + +createUser : CREATE USER user=userOrRoleName + ( IDENTIFIED BY password=literal )? ; + +setPassword : SET PASSWORD FOR user=userOrRoleName TO password=literal; + +dropUser : DROP USER user=userOrRoleName ; + +showUsers : SHOW USERS ; + +setRole : SET ROLE FOR user=userOrRoleName TO role=userOrRoleName; + +clearRole : CLEAR ROLE FOR user=userOrRoleName ; + +grantPrivilege : GRANT ( ALL PRIVILEGES | privileges=privilegeList ) TO userOrRole=userOrRoleName ; + +denyPrivilege : DENY ( ALL PRIVILEGES | privileges=privilegeList ) TO userOrRole=userOrRoleName ; + +revokePrivilege : REVOKE ( ALL PRIVILEGES | privileges=privilegeList ) FROM userOrRole=userOrRoleName ; + +privilege : CREATE + | DELETE + | MATCH + | MERGE + | SET + | REMOVE + | INDEX + | STATS + | AUTH + | CONSTRAINT + | DUMP + | REPLICATION + | READ_FILE + | FREE_MEMORY + | TRIGGER + | CONFIG + | DURABILITY + | STREAM + | MODULE_READ + | MODULE_WRITE + | WEBSOCKET + ; + +privilegeList : privilege ( ',' privilege )* ; + +showPrivileges : SHOW PRIVILEGES FOR userOrRole=userOrRoleName ; + +showRoleForUser : SHOW ROLE FOR user=userOrRoleName ; + +showUsersForRole : SHOW USERS FOR role=userOrRoleName ; + +dumpQuery: DUMP DATABASE ; + +setReplicationRole : SET REPLICATION ROLE TO ( MAIN | REPLICA ) + ( WITH PORT port=literal ) ? ; + +showReplicationRole : SHOW REPLICATION ROLE ; + +replicaName : symbolicName ; + +socketAddress : literal ; + +registerReplica : REGISTER REPLICA replicaName ( SYNC | ASYNC ) + ( WITH TIMEOUT timeout=literal ) ? + TO socketAddress ; + +dropReplica : DROP REPLICA replicaName ; + +showReplicas : SHOW REPLICAS ; + +lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY ; + +freeMemoryQuery : FREE MEMORY ; + +triggerName : symbolicName ; + +triggerStatement : .*? ; + +emptyVertex : '(' ')' ; + +emptyEdge : dash dash rightArrowHead ; + +createTrigger : CREATE TRIGGER triggerName ( ON ( emptyVertex | emptyEdge ) ? ( CREATE | UPDATE | DELETE ) ) ? + ( AFTER | BEFORE ) COMMIT EXECUTE triggerStatement ; + +dropTrigger : DROP TRIGGER triggerName ; + +showTriggers : SHOW TRIGGERS ; + +isolationLevel : SNAPSHOT ISOLATION | READ COMMITTED | READ UNCOMMITTED ; + +isolationLevelScope : GLOBAL | SESSION | NEXT ; + +isolationLevelQuery : SET isolationLevelScope TRANSACTION ISOLATION LEVEL isolationLevel ; + +createSnapshotQuery : CREATE SNAPSHOT ; + +streamName : symbolicName ; + +symbolicNameWithMinus : symbolicName ( MINUS symbolicName )* ; + +symbolicNameWithDotsAndMinus: symbolicNameWithMinus ( DOT symbolicNameWithMinus )* ; + +symbolicTopicNames : symbolicNameWithDotsAndMinus ( COMMA symbolicNameWithDotsAndMinus )* ; + +topicNames : symbolicTopicNames | literal ; + +commonCreateStreamConfig : TRANSFORM transformationName=procedureName + | BATCH_INTERVAL batchInterval=literal + | BATCH_SIZE batchSize=literal + ; + +createStream : kafkaCreateStream | pulsarCreateStream ; + +configKeyValuePair : literal ':' literal ; + +configMap : '{' ( configKeyValuePair ( ',' configKeyValuePair )* )? '}' ; + +kafkaCreateStreamConfig : TOPICS topicNames + | CONSUMER_GROUP consumerGroup=symbolicNameWithDotsAndMinus + | BOOTSTRAP_SERVERS bootstrapServers=literal + | CONFIGS configsMap=configMap + | CREDENTIALS credentialsMap=configMap + | commonCreateStreamConfig + ; + +kafkaCreateStream : CREATE KAFKA STREAM streamName ( kafkaCreateStreamConfig ) * ; + + +pulsarCreateStreamConfig : TOPICS topicNames + | SERVICE_URL serviceUrl=literal + | commonCreateStreamConfig + ; + +pulsarCreateStream : CREATE PULSAR STREAM streamName ( pulsarCreateStreamConfig ) * ; + +dropStream : DROP STREAM streamName ; + +startStream : START STREAM streamName ( BATCH_LIMIT batchLimit=literal ) ? ( TIMEOUT timeout=literal ) ? ; + +startAllStreams : START ALL STREAMS ; + +stopStream : STOP STREAM streamName ; + +stopAllStreams : STOP ALL STREAMS ; + +showStreams : SHOW STREAMS ; + +checkStream : CHECK STREAM streamName ( BATCH_LIMIT batchLimit=literal ) ? ( TIMEOUT timeout=literal ) ? ; + +settingName : literal ; + +settingValue : literal ; + +setSetting : SET DATABASE SETTING settingName TO settingValue ; + +showSetting : SHOW DATABASE SETTING settingName ; + +showSettings : SHOW DATABASE SETTINGS ; + +versionQuery : SHOW VERSION ; diff --git a/src/query/v2/frontend/opencypher/grammar/MemgraphCypherLexer.g4 b/src/query/v2/frontend/opencypher/grammar/MemgraphCypherLexer.g4 new file mode 100644 index 000000000..55e5d53a2 --- /dev/null +++ b/src/query/v2/frontend/opencypher/grammar/MemgraphCypherLexer.g4 @@ -0,0 +1,116 @@ +/* + * Copyright 2021 Memgraph Ltd. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source + * License, and you may not use this file except in compliance with the Business Source License. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +/* Memgraph specific Cypher reserved words used for enterprise features. */ + +/* + * When changing this grammar make sure to update constants in + * src/query/frontend/stripped_lexer_constants.hpp (kKeywords, kSpecialTokens + * and bitsets) if needed. + */ + +lexer grammar MemgraphCypherLexer ; + +import CypherLexer ; + +UNDERSCORE : '_' ; + +AFTER : A F T E R ; +ALTER : A L T E R ; +ASYNC : A S Y N C ; +AUTH : A U T H ; +BAD : B A D ; +BATCH_INTERVAL : B A T C H UNDERSCORE I N T E R V A L ; +BATCH_LIMIT : B A T C H UNDERSCORE L I M I T ; +BATCH_SIZE : B A T C H UNDERSCORE S I Z E ; +BEFORE : B E F O R E ; +BOOTSTRAP_SERVERS : B O O T S T R A P UNDERSCORE S E R V E R S ; +CHECK : C H E C K ; +CLEAR : C L E A R ; +COMMIT : C O M M I T ; +COMMITTED : C O M M I T T E D ; +CONFIG : C O N F I G ; +CONFIGS : C O N F I G S; +CONSUMER_GROUP : C O N S U M E R UNDERSCORE G R O U P ; +CREDENTIALS : C R E D E N T I A L S ; +CSV : C S V ; +DATA : D A T A ; +DELIMITER : D E L I M I T E R ; +DATABASE : D A T A B A S E ; +DENY : D E N Y ; +DIRECTORY : D I R E C T O R Y ; +DROP : D R O P ; +DUMP : D U M P ; +DURABILITY : D U R A B I L I T Y ; +EXECUTE : E X E C U T E ; +FOR : F O R ; +FOREACH : F O R E A C H; +FREE : F R E E ; +FREE_MEMORY : F R E E UNDERSCORE M E M O R Y ; +FROM : F R O M ; +GLOBAL : G L O B A L ; +GRANT : G R A N T ; +GRANTS : G R A N T S ; +HEADER : H E A D E R ; +IDENTIFIED : I D E N T I F I E D ; +IGNORE : I G N O R E ; +ISOLATION : I S O L A T I O N ; +KAFKA : K A F K A ; +LEVEL : L E V E L ; +LOAD : L O A D ; +LOCK : L O C K ; +MAIN : M A I N ; +MODE : M O D E ; +MODULE_READ : M O D U L E UNDERSCORE R E A D ; +MODULE_WRITE : M O D U L E UNDERSCORE W R I T E ; +NEXT : N E X T ; +NO : N O ; +PASSWORD : P A S S W O R D ; +PORT : P O R T ; +PRIVILEGES : P R I V I L E G E S ; +PULSAR : P U L S A R ; +READ : R E A D ; +READ_FILE : R E A D UNDERSCORE F I L E ; +REGISTER : R E G I S T E R ; +REPLICA : R E P L I C A ; +REPLICAS : R E P L I C A S ; +REPLICATION : R E P L I C A T I O N ; +REVOKE : R E V O K E ; +ROLE : R O L E ; +ROLES : R O L E S ; +QUOTE : Q U O T E ; +SERVICE_URL : S E R V I C E UNDERSCORE U R L ; +SESSION : S E S S I O N ; +SETTING : S E T T I N G ; +SETTINGS : S E T T I N G S ; +SNAPSHOT : S N A P S H O T ; +START : S T A R T ; +STATS : S T A T S ; +STOP : S T O P ; +STREAM : S T R E A M ; +STREAMS : S T R E A M S ; +SYNC : S Y N C ; +TIMEOUT : T I M E O U T ; +TO : T O ; +TOPICS : T O P I C S; +TRANSACTION : T R A N S A C T I O N ; +TRANSFORM : T R A N S F O R M ; +TRIGGER : T R I G G E R ; +TRIGGERS : T R I G G E R S ; +UNCOMMITTED : U N C O M M I T T E D ; +UNLOCK : U N L O C K ; +UPDATE : U P D A T E ; +USER : U S E R ; +USERS : U S E R S ; +VERSION : V E R S I O N ; +WEBSOCKET : W E B S O C K E T ; diff --git a/src/query/v2/frontend/opencypher/grammar/UnicodeCategories.g4 b/src/query/v2/frontend/opencypher/grammar/UnicodeCategories.g4 new file mode 100644 index 000000000..aa19ed96e --- /dev/null +++ b/src/query/v2/frontend/opencypher/grammar/UnicodeCategories.g4 @@ -0,0 +1,15 @@ +/** + * Unicode character categories used in openCypher lexer. This is separated from + * the lexer grammar as you probably don't want to ever change this (or even see + * this at all). + */ + +lexer grammar UnicodeCategories ; + +fragment ID_Start : [A-Za-z\u00AA\u00B5\u00BA\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376-\u0377\u037A-\u037D\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0620-\u064A\u066E-\u066F\u0671-\u06D3\u06D5\u06E5-\u06E6\u06EE-\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4-\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u08A0\u08A2-\u08AC\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097F\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC-\u09DD\u09DF-\u09E1\u09F0-\u09F1\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0-\u0AE1\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3D\u0B5C-\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C3D\u0C58-\u0C59\u0C60-\u0C61\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0-\u0CE1\u0CF1-\u0CF2\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D60-\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32-\u0E33\u0E40-\u0E46\u0E81-\u0E82\u0E84\u0E87-\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA-\u0EAB\u0EAD-\u0EB0\u0EB2-\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065-\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F4\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F0\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1877\u1880-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191C\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19C1-\u19C7\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4B\u1B83-\u1BA0\u1BAE-\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1CE9-\u1CEC\u1CEE-\u1CF1\u1CF5-\u1CF6\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2118-\u211D\u2124\u2126\u2128\u212A-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2160-\u2188\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CEE\u2CF2-\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303C\u3041-\u3096\u309B-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FCC\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A-\uA62B\uA640-\uA66E\uA67F-\uA697\uA6A0-\uA6EF\uA717-\uA71F\uA722-\uA788\uA78B-\uA78E\uA790-\uA793\uA7A0-\uA7AA\uA7F8-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA80-\uAAAF\uAAB1\uAAB5-\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uABC0-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC] ; + +fragment ID_Continue : [0-9A-Z_a-z\u00AA\u00B5\u00B7\u00BA\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0300-\u0374\u0376-\u0377\u037A-\u037D\u0386-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u0483-\u0487\u048A-\u0527\u0531-\u0556\u0559\u0561-\u0587\u0591-\u05BD\u05BF\u05C1-\u05C2\u05C4-\u05C5\u05C7\u05D0-\u05EA\u05F0-\u05F2\u0610-\u061A\u0620-\u0669\u066E-\u06D3\u06D5-\u06DC\u06DF-\u06E8\u06EA-\u06FC\u06FF\u0710-\u074A\u074D-\u07B1\u07C0-\u07F5\u07FA\u0800-\u082D\u0840-\u085B\u08A0\u08A2-\u08AC\u08E4-\u08FE\u0900-\u0963\u0966-\u096F\u0971-\u0977\u0979-\u097F\u0981-\u0983\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BC-\u09C4\u09C7-\u09C8\u09CB-\u09CE\u09D7\u09DC-\u09DD\u09DF-\u09E3\u09E6-\u09F1\u0A01-\u0A03\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A3C\u0A3E-\u0A42\u0A47-\u0A48\u0A4B-\u0A4D\u0A51\u0A59-\u0A5C\u0A5E\u0A66-\u0A75\u0A81-\u0A83\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABC-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AD0\u0AE0-\u0AE3\u0AE6-\u0AEF\u0B01-\u0B03\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3C-\u0B44\u0B47-\u0B48\u0B4B-\u0B4D\u0B56-\u0B57\u0B5C-\u0B5D\u0B5F-\u0B63\u0B66-\u0B6F\u0B71\u0B82-\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD0\u0BD7\u0BE6-\u0BEF\u0C01-\u0C03\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C3D-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55-\u0C56\u0C58-\u0C59\u0C60-\u0C63\u0C66-\u0C6F\u0C82-\u0C83\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBC-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5-\u0CD6\u0CDE\u0CE0-\u0CE3\u0CE6-\u0CEF\u0CF1-\u0CF2\u0D02-\u0D03\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D-\u0D44\u0D46-\u0D48\u0D4A-\u0D4E\u0D57\u0D60-\u0D63\u0D66-\u0D6F\u0D7A-\u0D7F\u0D82-\u0D83\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2-\u0DF3\u0E01-\u0E3A\u0E40-\u0E4E\u0E50-\u0E59\u0E81-\u0E82\u0E84\u0E87-\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA-\u0EAB\u0EAD-\u0EB9\u0EBB-\u0EBD\u0EC0-\u0EC4\u0EC6\u0EC8-\u0ECD\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F18-\u0F19\u0F20-\u0F29\u0F35\u0F37\u0F39\u0F3E-\u0F47\u0F49-\u0F6C\u0F71-\u0F84\u0F86-\u0F97\u0F99-\u0FBC\u0FC6\u1000-\u1049\u1050-\u109D\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u135D-\u135F\u1369-\u1371\u1380-\u138F\u13A0-\u13F4\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F0\u1700-\u170C\u170E-\u1714\u1720-\u1734\u1740-\u1753\u1760-\u176C\u176E-\u1770\u1772-\u1773\u1780-\u17D3\u17D7\u17DC-\u17DD\u17E0-\u17E9\u180B-\u180D\u1810-\u1819\u1820-\u1877\u1880-\u18AA\u18B0-\u18F5\u1900-\u191C\u1920-\u192B\u1930-\u193B\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A1B\u1A20-\u1A5E\u1A60-\u1A7C\u1A7F-\u1A89\u1A90-\u1A99\u1AA7\u1B00-\u1B4B\u1B50-\u1B59\u1B6B-\u1B73\u1B80-\u1BF3\u1C00-\u1C37\u1C40-\u1C49\u1C4D-\u1C7D\u1CD0-\u1CD2\u1CD4-\u1CF6\u1D00-\u1DE6\u1DFC-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u203F-\u2040\u2054\u2071\u207F\u2090-\u209C\u20D0-\u20DC\u20E1\u20E5-\u20F0\u2102\u2107\u210A-\u2113\u2115\u2118-\u211D\u2124\u2126\u2128\u212A-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2160-\u2188\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D7F-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2DE0-\u2DFF\u3005-\u3007\u3021-\u302F\u3031-\u3035\u3038-\u303C\u3041-\u3096\u3099-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FCC\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA66F\uA674-\uA67D\uA67F-\uA697\uA69F-\uA6F1\uA717-\uA71F\uA722-\uA788\uA78B-\uA78E\uA790-\uA793\uA7A0-\uA7AA\uA7F8-\uA827\uA840-\uA873\uA880-\uA8C4\uA8D0-\uA8D9\uA8E0-\uA8F7\uA8FB\uA900-\uA92D\uA930-\uA953\uA960-\uA97C\uA980-\uA9C0\uA9CF-\uA9D9\uAA00-\uAA36\uAA40-\uAA4D\uAA50-\uAA59\uAA60-\uAA76\uAA7A-\uAA7B\uAA80-\uAAC2\uAADB-\uAADD\uAAE0-\uAAEF\uAAF2-\uAAF6\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uABC0-\uABEA\uABEC-\uABED\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE00-\uFE0F\uFE20-\uFE26\uFE33-\uFE34\uFE4D-\uFE4F\uFE70-\uFE74\uFE76-\uFEFC\uFF10-\uFF19\uFF21-\uFF3A\uFF3F\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC] ; + +fragment Pc : [\u005F\u203F\u2040\u2054\uFE33\uFE34\uFE4D\uFE4E\uFE4F\uFF3F] ; + +fragment Sc : [$\u00A2-\u00A5\u058F\u060B\u09F2-\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BA\uA838\uFDFC\uFE69\uFF04\uFFE0-\uFFE1\uFFE5-\uFFE6] ; diff --git a/src/query/v2/frontend/opencypher/parser.hpp b/src/query/v2/frontend/opencypher/parser.hpp new file mode 100644 index 000000000..003209318 --- /dev/null +++ b/src/query/v2/frontend/opencypher/parser.hpp @@ -0,0 +1,68 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "antlr4-runtime.h" +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/opencypher/generated/MemgraphCypher.h" +#include "query/v2/frontend/opencypher/generated/MemgraphCypherLexer.h" + +namespace memgraph::query::v2::frontend::opencypher { + +/** + * Generates openCypher AST + * This thing must me a class since parser.cypher() returns pointer and there is + * no way for us to get ownership over the object. + */ +class Parser { + public: + /** + * @param query incoming query that has to be compiled into query plan + * the first step is to generate AST + */ + Parser(const std::string query) : query_(std::move(query)) { + parser_.removeErrorListeners(); + parser_.addErrorListener(&error_listener_); + tree_ = parser_.cypher(); + if (parser_.getNumberOfSyntaxErrors()) { + throw query::v2::SyntaxException(error_listener_.error_); + } + } + + auto tree() { return tree_; } + + private: + class FirstMessageErrorListener : public antlr4::BaseErrorListener { + void syntaxError(antlr4::Recognizer *, antlr4::Token *, size_t line, size_t position, const std::string &message, + std::exception_ptr) override { + if (error_.empty()) { + error_ = "line " + std::to_string(line) + ":" + std::to_string(position + 1) + " " + message; + } + } + + public: + std::string error_; + }; + + FirstMessageErrorListener error_listener_; + std::string query_; + antlr4::ANTLRInputStream input_{query_}; + antlropencypher::MemgraphCypherLexer lexer_{&input_}; + antlr4::CommonTokenStream tokens_{&lexer_}; + + // generate ast + antlropencypher::MemgraphCypher parser_{&tokens_}; + antlr4::tree::ParseTree *tree_ = nullptr; +}; +} // namespace memgraph::query::v2::frontend::opencypher diff --git a/src/query/v2/frontend/parsing.cpp b/src/query/v2/frontend/parsing.cpp new file mode 100644 index 000000000..1f3208d9a --- /dev/null +++ b/src/query/v2/frontend/parsing.cpp @@ -0,0 +1,184 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/frontend/parsing.hpp" + +#include +#include +#include +#include + +#include "query/v2/exceptions.hpp" +#include "utils/logging.hpp" +#include "utils/string.hpp" + +namespace memgraph::query::v2::frontend { + +int64_t ParseIntegerLiteral(const std::string &s) { + try { + // Not really correct since long long can have a bigger range than int64_t. + return static_cast(std::stoll(s, 0, 0)); + } catch (const std::out_of_range &) { + throw SemanticException("Integer literal exceeds 64 bits."); + } +} + +std::string ParseStringLiteral(const std::string &s) { + // These functions is declared as lambda since its semantics is highly + // specific for this conxtext and shouldn't be used elsewhere. + auto EncodeEscapedUnicodeCodepointUtf32 = [](const std::string &s, int &i) { + const int kLongUnicodeLength = 8; + int j = i + 1; + while (j < static_cast(s.size()) - 1 && j < i + kLongUnicodeLength + 1 && isxdigit(s[j])) { + ++j; + } + if (j - i == kLongUnicodeLength + 1) { + char32_t t = stoi(s.substr(i + 1, kLongUnicodeLength), 0, 16); + i += kLongUnicodeLength; + std::wstring_convert, char32_t> converter; + return converter.to_bytes(t); + } + throw SyntaxException( + "Expected 8 hex digits as unicode codepoint started with \\U. " + "Use \\u for 4 hex digits format."); + }; + auto EncodeEscapedUnicodeCodepointUtf16 = [](const std::string &s, int &i) { + const int kShortUnicodeLength = 4; + int j = i + 1; + while (j < static_cast(s.size()) - 1 && j < i + kShortUnicodeLength + 1 && isxdigit(s[j])) { + ++j; + } + if (j - i >= kShortUnicodeLength + 1) { + char16_t t = stoi(s.substr(i + 1, kShortUnicodeLength), 0, 16); + if (t >= 0xD800 && t <= 0xDBFF) { + // t is high surrogate pair. Expect one more utf16 codepoint. + j = i + kShortUnicodeLength + 1; + if (j >= static_cast(s.size()) - 1 || s[j] != '\\') { + throw SemanticException("Invalid UTF codepoint."); + } + ++j; + if (j >= static_cast(s.size()) - 1 || (s[j] != 'u' && s[j] != 'U')) { + throw SemanticException("Invalid UTF codepoint."); + } + ++j; + int k = j; + while (k < static_cast(s.size()) - 1 && k < j + kShortUnicodeLength && isxdigit(s[k])) { + ++k; + } + if (k != j + kShortUnicodeLength) { + throw SemanticException("Invalid UTF codepoint."); + } + char16_t surrogates[3] = {t, static_cast(stoi(s.substr(j, kShortUnicodeLength), 0, 16)), 0}; + i += kShortUnicodeLength + 2 + kShortUnicodeLength; + std::wstring_convert, char16_t> converter; + return converter.to_bytes(surrogates); + } else { + i += kShortUnicodeLength; + std::wstring_convert, char16_t> converter; + return converter.to_bytes(t); + } + } + throw SyntaxException( + "Expected 4 hex digits as unicode codepoint started with \\u. " + "Use \\U for 8 hex digits format."); + }; + + std::string unescaped; + bool escape = false; + + // First and last char is quote, we don't need to look at them. + for (int i = 1; i < static_cast(s.size()) - 1; ++i) { + if (escape) { + switch (s[i]) { + case '\\': + unescaped += '\\'; + break; + case '\'': + unescaped += '\''; + break; + case '"': + unescaped += '"'; + break; + case 'B': + case 'b': + unescaped += '\b'; + break; + case 'F': + case 'f': + unescaped += '\f'; + break; + case 'N': + case 'n': + unescaped += '\n'; + break; + case 'R': + case 'r': + unescaped += '\r'; + break; + case 'T': + case 't': + unescaped += '\t'; + break; + case 'U': + try { + unescaped += EncodeEscapedUnicodeCodepointUtf32(s, i); + } catch (const std::range_error &) { + throw SemanticException("Invalid UTF codepoint."); + } + break; + case 'u': + try { + unescaped += EncodeEscapedUnicodeCodepointUtf16(s, i); + } catch (const std::range_error &) { + throw SemanticException("Invalid UTF codepoint."); + } + break; + default: + // This should never happen, except grammar changes and we don't + // notice change in this production. + DLOG_FATAL("can't happen"); + throw std::exception(); + } + escape = false; + } else if (s[i] == '\\') { + escape = true; + } else { + unescaped += s[i]; + } + } + return unescaped; +} + +double ParseDoubleLiteral(const std::string &s) { + try { + return utils::ParseDouble(s); + } catch (const utils::BasicException &) { + throw SemanticException("Couldn't parse string to double."); + } +} + +std::string ParseParameter(const std::string &s) { + DMG_ASSERT(s[0] == '$', "Invalid string passed as parameter name"); + if (s[1] != '`') return s.substr(1); + // If parameter name is escaped symbolic name then symbolic name should be + // unescaped and leading and trailing backquote should be removed. + DMG_ASSERT(s.size() > 3U && s.back() == '`', "Invalid string passed as parameter name"); + std::string out; + for (int i = 2; i < static_cast(s.size()) - 1; ++i) { + if (s[i] == '`') { + ++i; + } + out.push_back(s[i]); + } + return out; +} + +} // namespace memgraph::query::v2::frontend diff --git a/src/query/v2/frontend/parsing.hpp b/src/query/v2/frontend/parsing.hpp new file mode 100644 index 000000000..2ba05b0d6 --- /dev/null +++ b/src/query/v2/frontend/parsing.hpp @@ -0,0 +1,27 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +/// @file +#pragma once + +#include +#include + +namespace memgraph::query::v2::frontend { + +// These are the functions for parsing literals and parameter names from +// opencypher query. +int64_t ParseIntegerLiteral(const std::string &s); +std::string ParseStringLiteral(const std::string &s); +double ParseDoubleLiteral(const std::string &s); +std::string ParseParameter(const std::string &s); + +} // namespace memgraph::query::v2::frontend diff --git a/src/query/v2/frontend/semantic/required_privileges.cpp b/src/query/v2/frontend/semantic/required_privileges.cpp new file mode 100644 index 000000000..0790529cf --- /dev/null +++ b/src/query/v2/frontend/semantic/required_privileges.cpp @@ -0,0 +1,152 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/ast/ast_visitor.hpp" +#include "query/v2/procedure/module.hpp" +#include "utils/memory.hpp" + +namespace memgraph::query::v2 { + +class PrivilegeExtractor : public QueryVisitor, public HierarchicalTreeVisitor { + public: + using HierarchicalTreeVisitor::PostVisit; + using HierarchicalTreeVisitor::PreVisit; + using HierarchicalTreeVisitor::Visit; + using QueryVisitor::Visit; + + std::vector privileges() { return privileges_; } + + void Visit(IndexQuery &) override { AddPrivilege(AuthQuery::Privilege::INDEX); } + + void Visit(AuthQuery &) override { AddPrivilege(AuthQuery::Privilege::AUTH); } + + void Visit(ExplainQuery &query) override { query.cypher_query_->Accept(*this); } + + void Visit(ProfileQuery &query) override { query.cypher_query_->Accept(*this); } + + void Visit(InfoQuery &info_query) override { + switch (info_query.info_type_) { + case InfoQuery::InfoType::INDEX: + // TODO: This should be INDEX | STATS, but we don't have support for + // *or* with privileges. + AddPrivilege(AuthQuery::Privilege::INDEX); + break; + case InfoQuery::InfoType::STORAGE: + AddPrivilege(AuthQuery::Privilege::STATS); + break; + case InfoQuery::InfoType::CONSTRAINT: + // TODO: This should be CONSTRAINT | STATS, but we don't have support + // for *or* with privileges. + AddPrivilege(AuthQuery::Privilege::CONSTRAINT); + break; + } + } + + void Visit(ConstraintQuery &constraint_query) override { AddPrivilege(AuthQuery::Privilege::CONSTRAINT); } + + void Visit(CypherQuery &query) override { + query.single_query_->Accept(*this); + for (auto *cypher_union : query.cypher_unions_) { + cypher_union->Accept(*this); + } + } + + void Visit(DumpQuery &dump_query) override { AddPrivilege(AuthQuery::Privilege::DUMP); } + + void Visit(LockPathQuery &lock_path_query) override { AddPrivilege(AuthQuery::Privilege::DURABILITY); } + + void Visit(FreeMemoryQuery &free_memory_query) override { AddPrivilege(AuthQuery::Privilege::FREE_MEMORY); } + + void Visit(TriggerQuery &trigger_query) override { AddPrivilege(AuthQuery::Privilege::TRIGGER); } + + void Visit(StreamQuery &stream_query) override { AddPrivilege(AuthQuery::Privilege::STREAM); } + + void Visit(ReplicationQuery &replication_query) override { AddPrivilege(AuthQuery::Privilege::REPLICATION); } + + void Visit(IsolationLevelQuery &isolation_level_query) override { AddPrivilege(AuthQuery::Privilege::CONFIG); } + + void Visit(CreateSnapshotQuery &create_snapshot_query) override { AddPrivilege(AuthQuery::Privilege::DURABILITY); } + + void Visit(SettingQuery & /*setting_query*/) override { AddPrivilege(AuthQuery::Privilege::CONFIG); } + + void Visit(VersionQuery & /*version_query*/) override { AddPrivilege(AuthQuery::Privilege::STATS); } + + bool PreVisit(Create & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::CREATE); + return false; + } + bool PreVisit(CallProcedure &procedure) override { + const auto maybe_proc = + procedure::FindProcedure(procedure::gModuleRegistry, procedure.procedure_name_, utils::NewDeleteResource()); + if (maybe_proc && maybe_proc->second->info.required_privilege) { + AddPrivilege(*maybe_proc->second->info.required_privilege); + } + return false; + } + bool PreVisit(Delete & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::DELETE); + return false; + } + bool PreVisit(Match & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::MATCH); + return false; + } + bool PreVisit(Merge & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::MERGE); + return false; + } + bool PreVisit(SetProperty & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::SET); + return false; + } + bool PreVisit(SetProperties & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::SET); + return false; + } + bool PreVisit(SetLabels & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::SET); + return false; + } + bool PreVisit(RemoveProperty & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::REMOVE); + return false; + } + bool PreVisit(RemoveLabels & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::REMOVE); + return false; + } + bool PreVisit(LoadCsv & /*unused*/) override { + AddPrivilege(AuthQuery::Privilege::READ_FILE); + return false; + } + + bool Visit(Identifier & /*unused*/) override { return true; } + bool Visit(PrimitiveLiteral & /*unused*/) override { return true; } + bool Visit(ParameterLookup & /*unused*/) override { return true; } + + private: + void AddPrivilege(AuthQuery::Privilege privilege) { + if (!utils::Contains(privileges_, privilege)) { + privileges_.push_back(privilege); + } + } + + std::vector privileges_; +}; + +std::vector GetRequiredPrivileges(Query *query) { + PrivilegeExtractor extractor; + query->Accept(extractor); + return extractor.privileges(); +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/semantic/required_privileges.hpp b/src/query/v2/frontend/semantic/required_privileges.hpp new file mode 100644 index 000000000..943f786a3 --- /dev/null +++ b/src/query/v2/frontend/semantic/required_privileges.hpp @@ -0,0 +1,18 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "query/v2/frontend/ast/ast.hpp" + +namespace memgraph::query::v2 { +std::vector GetRequiredPrivileges(Query *query); +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/semantic/symbol.lcp b/src/query/v2/frontend/semantic/symbol.lcp new file mode 100644 index 000000000..c5b0b8030 --- /dev/null +++ b/src/query/v2/frontend/semantic/symbol.lcp @@ -0,0 +1,89 @@ +;; Copyright 2022 Memgraph Ltd. +;; +;; Use of this software is governed by the Business Source License +;; included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +;; License, and you may not use this file except in compliance with the Business Source License. +;; +;; As of the Change Date specified in that file, in accordance with +;; the Business Source License, use of this software will be governed +;; by the Apache License, Version 2.0, included in the file +;; licenses/APL.txt. + +#>cpp +#pragma once + +#include + +#include "utils/typeinfo.hpp" +cpp<# + +(lcp:namespace memgraph) +(lcp:namespace query) +(lcp:namespace v2) + +(lcp:define-class symbol () + ((name "std::string" :scope :public) + (position :int64_t :scope :public) + (user-declared :bool :initval "true" :scope :public) + (type "Type" :initval "Type::ANY" :scope :public) + (token-position :int64_t :initval "-1" :scope :public)) + (:public + ;; This is similar to TypedValue::Type, but this has `Any` type. + ;; TODO: Make a better Type structure which can store a generic List. + (lcp:define-enum type (any vertex edge path number edge-list) + (:serialize)) + #>cpp + // TODO: Generate enum to string conversion from LCP. Note, that this is + // displayed to the end user, so we may want to have a pretty name of each + // value. + static std::string TypeToString(Type type) { + const char *enum_string[] = {"Any", "Vertex", "Edge", + "Path", "Number", "EdgeList"}; + return enum_string[static_cast(type)]; + } + + Symbol() {} + Symbol(const std::string &name, int position, bool user_declared, + Type type = Type::ANY, int token_position = -1) + : name_(name), + position_(position), + user_declared_(user_declared), + type_(type), + token_position_(token_position) {} + + bool operator==(const Symbol &other) const { + return position_ == other.position_ && name_ == other.name_ && + type_ == other.type_; + } + bool operator!=(const Symbol &other) const { return !operator==(other); } + + // TODO: Remove these since members are public + const auto &name() const { return name_; } + int position() const { return position_; } + Type type() const { return type_; } + bool user_declared() const { return user_declared_; } + int token_position() const { return token_position_; } + cpp<#) + (:serialize (:slk))) + +(lcp:pop-namespace) ;; v2 +(lcp:pop-namespace) ;; query +(lcp:pop-namespace) ;; memgraph + +#>cpp +namespace std { + +template <> +struct hash { + size_t operator()(const memgraph::query::v2::Symbol &symbol) const { + size_t prime = 265443599u; + size_t hash = std::hash{}(symbol.position()); + hash ^= prime * std::hash{}(symbol.name()); + hash ^= prime * std::hash{}(static_cast(symbol.type())); + return hash; + } +}; + +} // namespace std + +cpp<# diff --git a/src/query/v2/frontend/semantic/symbol_generator.cpp b/src/query/v2/frontend/semantic/symbol_generator.cpp new file mode 100644 index 000000000..64e3604b1 --- /dev/null +++ b/src/query/v2/frontend/semantic/symbol_generator.cpp @@ -0,0 +1,625 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +// Copyright 2017 Memgraph +// +// Created by Teon Banek on 24-03-2017 + +#include "query/v2/frontend/semantic/symbol_generator.hpp" + +#include +#include +#include +#include +#include + +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/ast/ast_visitor.hpp" +#include "utils/algorithm.hpp" +#include "utils/logging.hpp" + +namespace memgraph::query::v2 { + +namespace { +std::unordered_map GeneratePredefinedIdentifierMap( + const std::vector &predefined_identifiers) { + std::unordered_map identifier_map; + for (const auto &identifier : predefined_identifiers) { + identifier_map.emplace(identifier->name_, identifier); + } + + return identifier_map; +} +} // namespace + +SymbolGenerator::SymbolGenerator(SymbolTable *symbol_table, const std::vector &predefined_identifiers) + : symbol_table_(symbol_table), + predefined_identifiers_{GeneratePredefinedIdentifierMap(predefined_identifiers)}, + scopes_(1, Scope()) {} + +std::optional SymbolGenerator::FindSymbolInScope(const std::string &name, const Scope &scope, + Symbol::Type type) { + if (auto it = scope.symbols.find(name); it != scope.symbols.end()) { + const auto &symbol = it->second; + // Unless we have `ANY` type, check that types match. + if (type != Symbol::Type::ANY && symbol.type() != Symbol::Type::ANY && type != symbol.type()) { + throw TypeMismatchError(name, Symbol::TypeToString(symbol.type()), Symbol::TypeToString(type)); + } + return symbol; + } + return std::nullopt; +} + +auto SymbolGenerator::CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type, int token_position) { + auto symbol = symbol_table_->CreateSymbol(name, user_declared, type, token_position); + scopes_.back().symbols[name] = symbol; + return symbol; +} + +auto SymbolGenerator::GetOrCreateSymbolLocalScope(const std::string &name, bool user_declared, Symbol::Type type) { + auto &scope = scopes_.back(); + if (auto maybe_symbol = FindSymbolInScope(name, scope, type); maybe_symbol) { + return *maybe_symbol; + } + return CreateSymbol(name, user_declared, type); +} + +auto SymbolGenerator::GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type) { + // NOLINTNEXTLINE + for (auto scope = scopes_.rbegin(); scope != scopes_.rend(); ++scope) { + if (auto maybe_symbol = FindSymbolInScope(name, *scope, type); maybe_symbol) { + return *maybe_symbol; + } + } + return CreateSymbol(name, user_declared, type); +} + +void SymbolGenerator::VisitReturnBody(ReturnBody &body, Where *where) { + auto &scope = scopes_.back(); + for (auto &expr : body.named_expressions) { + expr->Accept(*this); + } + std::vector user_symbols; + if (body.all_identifiers) { + // Carry over user symbols because '*' appeared. + for (const auto &sym_pair : scope.symbols) { + if (!sym_pair.second.user_declared()) { + continue; + } + user_symbols.emplace_back(sym_pair.second); + } + if (user_symbols.empty()) { + throw SemanticException("There are no variables in scope to use for '*'."); + } + } + // WITH/RETURN clause removes declarations of all the previous variables and + // declares only those established through named expressions. New declarations + // must not be visible inside named expressions themselves. + bool removed_old_names = false; + if ((!where && body.order_by.empty()) || scope.has_aggregation) { + // WHERE and ORDER BY need to see both the old and new symbols, unless we + // have an aggregation. Therefore, we can clear the symbols immediately if + // there is neither ORDER BY nor WHERE, or we have an aggregation. + scope.symbols.clear(); + removed_old_names = true; + } + // Create symbols for named expressions. + std::unordered_set new_names; + for (const auto &user_sym : user_symbols) { + new_names.insert(user_sym.name()); + scope.symbols[user_sym.name()] = user_sym; + } + for (auto &named_expr : body.named_expressions) { + const auto &name = named_expr->name_; + if (!new_names.insert(name).second) { + throw SemanticException("Multiple results with the same name '{}' are not allowed.", name); + } + // An improvement would be to infer the type of the expression, so that the + // new symbol would have a more specific type. + named_expr->MapTo(CreateSymbol(name, true, Symbol::Type::ANY, named_expr->token_position_)); + } + scope.in_order_by = true; + for (const auto &order_pair : body.order_by) { + order_pair.expression->Accept(*this); + } + scope.in_order_by = false; + if (body.skip) { + scope.in_skip = true; + body.skip->Accept(*this); + scope.in_skip = false; + } + if (body.limit) { + scope.in_limit = true; + body.limit->Accept(*this); + scope.in_limit = false; + } + if (where) where->Accept(*this); + if (!removed_old_names) { + // We have an ORDER BY or WHERE, but no aggregation, which means we didn't + // clear the old symbols, so do it now. We cannot just call clear, because + // we've added new symbols. + for (auto sym_it = scope.symbols.begin(); sym_it != scope.symbols.end();) { + if (new_names.find(sym_it->first) == new_names.end()) { + sym_it = scope.symbols.erase(sym_it); + } else { + sym_it++; + } + } + } + scopes_.back().has_aggregation = false; +} + +// Query + +bool SymbolGenerator::PreVisit(SingleQuery &) { + prev_return_names_ = curr_return_names_; + curr_return_names_.clear(); + return true; +} + +// Union + +bool SymbolGenerator::PreVisit(CypherUnion &) { + scopes_.back() = Scope(); + return true; +} + +bool SymbolGenerator::PostVisit(CypherUnion &cypher_union) { + if (prev_return_names_ != curr_return_names_) { + throw SemanticException("All subqueries in an UNION must have the same column names."); + } + + // create new symbols for the result of the union + for (const auto &name : curr_return_names_) { + auto symbol = CreateSymbol(name, false); + cypher_union.union_symbols_.push_back(symbol); + } + + return true; +} + +// Clauses + +bool SymbolGenerator::PreVisit(Create &) { + scopes_.back().in_create = true; + return true; +} +bool SymbolGenerator::PostVisit(Create &) { + scopes_.back().in_create = false; + return true; +} + +bool SymbolGenerator::PreVisit(CallProcedure &call_proc) { + for (auto *expr : call_proc.arguments_) { + expr->Accept(*this); + } + return false; +} + +bool SymbolGenerator::PostVisit(CallProcedure &call_proc) { + for (auto *ident : call_proc.result_identifiers_) { + if (HasSymbolLocalScope(ident->name_)) { + throw RedeclareVariableError(ident->name_); + } + ident->MapTo(CreateSymbol(ident->name_, true)); + } + return true; +} + +bool SymbolGenerator::PreVisit(LoadCsv &load_csv) { return false; } + +bool SymbolGenerator::PostVisit(LoadCsv &load_csv) { + if (HasSymbolLocalScope(load_csv.row_var_->name_)) { + throw RedeclareVariableError(load_csv.row_var_->name_); + } + load_csv.row_var_->MapTo(CreateSymbol(load_csv.row_var_->name_, true)); + return true; +} + +bool SymbolGenerator::PreVisit(Return &ret) { + auto &scope = scopes_.back(); + scope.in_return = true; + VisitReturnBody(ret.body_); + scope.in_return = false; + return false; // We handled the traversal ourselves. +} + +bool SymbolGenerator::PostVisit(Return &) { + for (const auto &name_symbol : scopes_.back().symbols) curr_return_names_.insert(name_symbol.first); + return true; +} + +bool SymbolGenerator::PreVisit(With &with) { + auto &scope = scopes_.back(); + scope.in_with = true; + VisitReturnBody(with.body_, with.where_); + scope.in_with = false; + return false; // We handled the traversal ourselves. +} + +bool SymbolGenerator::PreVisit(Where &) { + scopes_.back().in_where = true; + return true; +} +bool SymbolGenerator::PostVisit(Where &) { + scopes_.back().in_where = false; + return true; +} + +bool SymbolGenerator::PreVisit(Merge &) { + scopes_.back().in_merge = true; + return true; +} +bool SymbolGenerator::PostVisit(Merge &) { + scopes_.back().in_merge = false; + return true; +} + +bool SymbolGenerator::PostVisit(Unwind &unwind) { + const auto &name = unwind.named_expression_->name_; + if (HasSymbolLocalScope(name)) { + throw RedeclareVariableError(name); + } + unwind.named_expression_->MapTo(CreateSymbol(name, true)); + return true; +} + +bool SymbolGenerator::PreVisit(Match &) { + scopes_.back().in_match = true; + return true; +} +bool SymbolGenerator::PostVisit(Match &) { + auto &scope = scopes_.back(); + scope.in_match = false; + // Check variables in property maps after visiting Match, so that they can + // reference symbols out of bind order. + for (auto &ident : scope.identifiers_in_match) { + if (!HasSymbolLocalScope(ident->name_) && !ConsumePredefinedIdentifier(ident->name_)) + throw UnboundVariableError(ident->name_); + ident->MapTo(scope.symbols[ident->name_]); + } + scope.identifiers_in_match.clear(); + return true; +} + +bool SymbolGenerator::PreVisit(Foreach &for_each) { + const auto &name = for_each.named_expression_->name_; + scopes_.emplace_back(Scope()); + scopes_.back().in_foreach = true; + for_each.named_expression_->MapTo( + CreateSymbol(name, true, Symbol::Type::ANY, for_each.named_expression_->token_position_)); + return true; +} +bool SymbolGenerator::PostVisit([[maybe_unused]] Foreach &for_each) { + scopes_.pop_back(); + return true; +} + +// Expressions + +SymbolGenerator::ReturnType SymbolGenerator::Visit(Identifier &ident) { + auto &scope = scopes_.back(); + if (scope.in_skip || scope.in_limit) { + throw SemanticException("Variables are not allowed in {}.", scope.in_skip ? "SKIP" : "LIMIT"); + } + Symbol symbol; + if (scope.in_pattern && !(scope.in_node_atom || scope.visiting_edge)) { + // If we are in the pattern, and outside of a node or an edge, the + // identifier is the pattern name. + symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, Symbol::Type::PATH); + } else if (scope.in_pattern && scope.in_pattern_atom_identifier) { + // Patterns used to create nodes and edges cannot redeclare already + // established bindings. Declaration only happens in single node + // patterns and in edge patterns. OpenCypher example, + // `MATCH (n) CREATE (n)` should throw an error that `n` is already + // declared. While `MATCH (n) CREATE (n) -[:R]-> (n)` is allowed, + // since `n` now references the bound node instead of declaring it. + if ((scope.in_create_node || scope.in_create_edge) && HasSymbolLocalScope(ident.name_)) { + throw RedeclareVariableError(ident.name_); + } + auto type = Symbol::Type::VERTEX; + if (scope.visiting_edge) { + // Edge referencing is not allowed (like in Neo4j): + // `MATCH (n) - [r] -> (n) - [r] -> (n) RETURN r` is not allowed. + if (HasSymbolLocalScope(ident.name_)) { + throw RedeclareVariableError(ident.name_); + } + type = scope.visiting_edge->IsVariable() ? Symbol::Type::EDGE_LIST : Symbol::Type::EDGE; + } + symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, type); + } else if (scope.in_pattern && !scope.in_pattern_atom_identifier && scope.in_match) { + if (scope.in_edge_range && scope.visiting_edge->identifier_->name_ == ident.name_) { + // Prevent variable path bounds to reference the identifier which is bound + // by the variable path itself. + throw UnboundVariableError(ident.name_); + } + // Variables in property maps or bounds of variable length path during MATCH + // can reference symbols bound later in the same MATCH. We collect them + // here, so that they can be checked after visiting Match. + scope.identifiers_in_match.emplace_back(&ident); + } else { + // Everything else references a bound symbol. + if (!HasSymbol(ident.name_) && !ConsumePredefinedIdentifier(ident.name_)) throw UnboundVariableError(ident.name_); + symbol = GetOrCreateSymbol(ident.name_, ident.user_declared_, Symbol::Type::ANY); + } + ident.MapTo(symbol); + return true; +} + +bool SymbolGenerator::PreVisit(Aggregation &aggr) { + auto &scope = scopes_.back(); + // Check if the aggregation can be used in this context. This check should + // probably move to a separate phase, which checks if the query is well + // formed. + if ((!scope.in_return && !scope.in_with) || scope.in_order_by || scope.in_skip || scope.in_limit || scope.in_where) { + throw SemanticException("Aggregation functions are only allowed in WITH and RETURN."); + } + if (scope.in_aggregation) { + throw SemanticException( + "Using aggregation functions inside aggregation functions is not " + "allowed."); + } + if (scope.num_if_operators) { + // Neo allows aggregations here and produces very interesting behaviors. + // To simplify implementation at this moment we decided to completely + // disallow aggregations inside of the CASE. + // However, in some cases aggregation makes perfect sense, for example: + // CASE count(n) WHEN 10 THEN "YES" ELSE "NO" END. + // TODO: Rethink of allowing aggregations in some parts of the CASE + // construct. + throw SemanticException("Using aggregation functions inside of CASE is not allowed."); + } + // Create a virtual symbol for aggregation result. + // Currently, we only have aggregation operators which return numbers. + auto aggr_name = Aggregation::OpToString(aggr.op_) + std::to_string(aggr.symbol_pos_); + aggr.MapTo(CreateSymbol(aggr_name, false, Symbol::Type::NUMBER)); + scope.in_aggregation = true; + scope.has_aggregation = true; + return true; +} + +bool SymbolGenerator::PostVisit(Aggregation &) { + scopes_.back().in_aggregation = false; + return true; +} + +bool SymbolGenerator::PreVisit(IfOperator &) { + ++scopes_.back().num_if_operators; + return true; +} + +bool SymbolGenerator::PostVisit(IfOperator &) { + --scopes_.back().num_if_operators; + return true; +} + +bool SymbolGenerator::PreVisit(All &all) { + all.list_expression_->Accept(*this); + VisitWithIdentifiers(all.where_->expression_, {all.identifier_}); + return false; +} + +bool SymbolGenerator::PreVisit(Single &single) { + single.list_expression_->Accept(*this); + VisitWithIdentifiers(single.where_->expression_, {single.identifier_}); + return false; +} + +bool SymbolGenerator::PreVisit(Any &any) { + any.list_expression_->Accept(*this); + VisitWithIdentifiers(any.where_->expression_, {any.identifier_}); + return false; +} + +bool SymbolGenerator::PreVisit(None &none) { + none.list_expression_->Accept(*this); + VisitWithIdentifiers(none.where_->expression_, {none.identifier_}); + return false; +} + +bool SymbolGenerator::PreVisit(Reduce &reduce) { + reduce.initializer_->Accept(*this); + reduce.list_->Accept(*this); + VisitWithIdentifiers(reduce.expression_, {reduce.accumulator_, reduce.identifier_}); + return false; +} + +bool SymbolGenerator::PreVisit(Extract &extract) { + extract.list_->Accept(*this); + VisitWithIdentifiers(extract.expression_, {extract.identifier_}); + return false; +} + +// Pattern and its subparts. + +bool SymbolGenerator::PreVisit(Pattern &pattern) { + auto &scope = scopes_.back(); + scope.in_pattern = true; + if ((scope.in_create || scope.in_merge) && pattern.atoms_.size() == 1U) { + MG_ASSERT(utils::IsSubtype(*pattern.atoms_[0], NodeAtom::kType), "Expected a single NodeAtom in Pattern"); + scope.in_create_node = true; + } + return true; +} + +bool SymbolGenerator::PostVisit(Pattern &) { + auto &scope = scopes_.back(); + scope.in_pattern = false; + scope.in_create_node = false; + return true; +} + +bool SymbolGenerator::PreVisit(NodeAtom &node_atom) { + auto &scope = scopes_.back(); + auto check_node_semantic = [&node_atom, &scope, this](const bool props_or_labels) { + const auto &node_name = node_atom.identifier_->name_; + if ((scope.in_create || scope.in_merge) && props_or_labels && HasSymbolLocalScope(node_name)) { + throw SemanticException("Cannot create node '" + node_name + + "' with labels or properties, because it is already declared."); + } + scope.in_pattern_atom_identifier = true; + node_atom.identifier_->Accept(*this); + scope.in_pattern_atom_identifier = false; + }; + + scope.in_node_atom = true; + if (auto *properties = std::get_if>(&node_atom.properties_)) { + bool props_or_labels = !properties->empty() || !node_atom.labels_.empty(); + + check_node_semantic(props_or_labels); + for (auto kv : *properties) { + kv.second->Accept(*this); + } + + return false; + } + auto &properties_parameter = std::get(node_atom.properties_); + bool props_or_labels = !properties_parameter || !node_atom.labels_.empty(); + + check_node_semantic(props_or_labels); + properties_parameter->Accept(*this); + return false; +} + +bool SymbolGenerator::PostVisit(NodeAtom &) { + scopes_.back().in_node_atom = false; + return true; +} + +bool SymbolGenerator::PreVisit(EdgeAtom &edge_atom) { + auto &scope = scopes_.back(); + scope.visiting_edge = &edge_atom; + if (scope.in_create || scope.in_merge) { + scope.in_create_edge = true; + if (edge_atom.edge_types_.size() != 1U) { + throw SemanticException( + "A single relationship type must be specified " + "when creating an edge."); + } + if (scope.in_create && // Merge allows bidirectionality + edge_atom.direction_ == EdgeAtom::Direction::BOTH) { + throw SemanticException( + "Bidirectional relationship are not supported " + "when creating an edge"); + } + if (edge_atom.IsVariable()) { + throw SemanticException( + "Variable length relationships are not supported when creating an " + "edge."); + } + } + if (auto *properties = std::get_if>(&edge_atom.properties_)) { + for (auto kv : *properties) { + kv.second->Accept(*this); + } + } else { + std::get(edge_atom.properties_)->Accept(*this); + } + if (edge_atom.IsVariable()) { + scope.in_edge_range = true; + if (edge_atom.lower_bound_) { + edge_atom.lower_bound_->Accept(*this); + } + if (edge_atom.upper_bound_) { + edge_atom.upper_bound_->Accept(*this); + } + scope.in_edge_range = false; + scope.in_pattern = false; + if (edge_atom.filter_lambda_.expression) { + VisitWithIdentifiers(edge_atom.filter_lambda_.expression, + {edge_atom.filter_lambda_.inner_edge, edge_atom.filter_lambda_.inner_node}); + } else { + // Create inner symbols, but don't bind them in scope, since they are to + // be used in the missing filter expression. + auto *inner_edge = edge_atom.filter_lambda_.inner_edge; + inner_edge->MapTo(symbol_table_->CreateSymbol(inner_edge->name_, inner_edge->user_declared_, Symbol::Type::EDGE)); + auto *inner_node = edge_atom.filter_lambda_.inner_node; + inner_node->MapTo( + symbol_table_->CreateSymbol(inner_node->name_, inner_node->user_declared_, Symbol::Type::VERTEX)); + } + if (edge_atom.weight_lambda_.expression) { + VisitWithIdentifiers(edge_atom.weight_lambda_.expression, + {edge_atom.weight_lambda_.inner_edge, edge_atom.weight_lambda_.inner_node}); + } + scope.in_pattern = true; + } + scope.in_pattern_atom_identifier = true; + edge_atom.identifier_->Accept(*this); + scope.in_pattern_atom_identifier = false; + if (edge_atom.total_weight_) { + if (HasSymbolLocalScope(edge_atom.total_weight_->name_)) { + throw RedeclareVariableError(edge_atom.total_weight_->name_); + } + edge_atom.total_weight_->MapTo(GetOrCreateSymbolLocalScope( + edge_atom.total_weight_->name_, edge_atom.total_weight_->user_declared_, Symbol::Type::NUMBER)); + } + return false; +} + +bool SymbolGenerator::PostVisit(EdgeAtom &) { + auto &scope = scopes_.back(); + scope.visiting_edge = nullptr; + scope.in_create_edge = false; + return true; +} + +void SymbolGenerator::VisitWithIdentifiers(Expression *expr, const std::vector &identifiers) { + auto &scope = scopes_.back(); + std::vector, Identifier *>> prev_symbols; + // Collect previous symbols if they exist. + for (const auto &identifier : identifiers) { + std::optional prev_symbol; + auto prev_symbol_it = scope.symbols.find(identifier->name_); + if (prev_symbol_it != scope.symbols.end()) { + prev_symbol = prev_symbol_it->second; + } + identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_)); + prev_symbols.emplace_back(prev_symbol, identifier); + } + // Visit the expression with the new symbols bound. + expr->Accept(*this); + // Restore back to previous symbols. + for (const auto &prev : prev_symbols) { + const auto &prev_symbol = prev.first; + const auto &identifier = prev.second; + if (prev_symbol) { + scope.symbols[identifier->name_] = *prev_symbol; + } else { + scope.symbols.erase(identifier->name_); + } + } +} + +bool SymbolGenerator::HasSymbol(const std::string &name) const { + return std::ranges::any_of(scopes_, [&name](const auto &scope) { return scope.symbols.contains(name); }); +} + +bool SymbolGenerator::HasSymbolLocalScope(const std::string &name) const { + return scopes_.back().symbols.contains(name); +} + +bool SymbolGenerator::ConsumePredefinedIdentifier(const std::string &name) { + auto it = predefined_identifiers_.find(name); + + if (it == predefined_identifiers_.end()) { + return false; + } + + // we can only use the predefined identifier in a single scope so we remove it after creating + // a symbol for it + auto &identifier = it->second; + MG_ASSERT(!identifier->user_declared_, "Predefined symbols cannot be user declared!"); + identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_)); + predefined_identifiers_.erase(it); + return true; +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/semantic/symbol_generator.hpp b/src/query/v2/frontend/semantic/symbol_generator.hpp new file mode 100644 index 000000000..991717bdf --- /dev/null +++ b/src/query/v2/frontend/semantic/symbol_generator.hpp @@ -0,0 +1,176 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +// Copyright 2017 Memgraph +// +// Created by Teon Banek on 11-03-2017 + +#pragma once + +#include +#include + +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/semantic/symbol_table.hpp" + +namespace memgraph::query::v2 { + +/// Visits the AST and generates symbols for variables. +/// +/// During the process of symbol generation, simple semantic checks are +/// performed. Such as, redeclaring a variable or conflicting expectations of +/// variable types. +class SymbolGenerator : public HierarchicalTreeVisitor { + public: + explicit SymbolGenerator(SymbolTable *symbol_table, const std::vector &predefined_identifiers); + + using HierarchicalTreeVisitor::PostVisit; + using HierarchicalTreeVisitor::PreVisit; + using HierarchicalTreeVisitor::Visit; + using typename HierarchicalTreeVisitor::ReturnType; + + // Query + bool PreVisit(SingleQuery &) override; + + // Union + bool PreVisit(CypherUnion &) override; + bool PostVisit(CypherUnion &) override; + + // Clauses + bool PreVisit(Create &) override; + bool PostVisit(Create &) override; + bool PreVisit(CallProcedure &) override; + bool PostVisit(CallProcedure &) override; + bool PreVisit(LoadCsv &) override; + bool PostVisit(LoadCsv &) override; + bool PreVisit(Return &) override; + bool PostVisit(Return &) override; + bool PreVisit(With &) override; + bool PreVisit(Where &) override; + bool PostVisit(Where &) override; + bool PreVisit(Merge &) override; + bool PostVisit(Merge &) override; + bool PostVisit(Unwind &) override; + bool PreVisit(Match &) override; + bool PostVisit(Match &) override; + bool PreVisit(Foreach &) override; + bool PostVisit(Foreach &) override; + + // Expressions + ReturnType Visit(Identifier &) override; + ReturnType Visit(PrimitiveLiteral &) override { return true; } + ReturnType Visit(ParameterLookup &) override { return true; } + bool PreVisit(Aggregation &) override; + bool PostVisit(Aggregation &) override; + bool PreVisit(IfOperator &) override; + bool PostVisit(IfOperator &) override; + bool PreVisit(All &) override; + bool PreVisit(Single &) override; + bool PreVisit(Any &) override; + bool PreVisit(None &) override; + bool PreVisit(Reduce &) override; + bool PreVisit(Extract &) override; + + // Pattern and its subparts. + bool PreVisit(Pattern &) override; + bool PostVisit(Pattern &) override; + bool PreVisit(NodeAtom &) override; + bool PostVisit(NodeAtom &) override; + bool PreVisit(EdgeAtom &) override; + bool PostVisit(EdgeAtom &) override; + + private: + // Scope stores the state of where we are when visiting the AST and a map of + // names to symbols. + struct Scope { + bool in_pattern{false}; + bool in_merge{false}; + bool in_create{false}; + // in_create_node is true if we are creating or merging *only* a node. + // Therefore, it is *not* equivalent to (in_create || in_merge) && + // in_node_atom. + bool in_create_node{false}; + // True if creating an edge; + // shortcut for (in_create || in_merge) && visiting_edge. + bool in_create_edge{false}; + bool in_node_atom{false}; + EdgeAtom *visiting_edge{nullptr}; + bool in_aggregation{false}; + bool in_return{false}; + bool in_with{false}; + bool in_skip{false}; + bool in_limit{false}; + bool in_order_by{false}; + bool in_where{false}; + bool in_match{false}; + bool in_foreach{false}; + // True when visiting a pattern atom (node or edge) identifier, which can be + // reused or created in the pattern itself. + bool in_pattern_atom_identifier{false}; + // True when visiting range bounds of a variable path. + bool in_edge_range{false}; + // True if the return/with contains an aggregation in any named expression. + bool has_aggregation{false}; + // Map from variable names to symbols. + std::map symbols; + // Identifiers found in property maps of patterns or as variable length path + // bounds in a single Match clause. They need to be checked after visiting + // Match. Identifiers created by naming vertices, edges and paths are *not* + // stored in here. + std::vector identifiers_in_match; + // Number of nested IfOperators. + int num_if_operators{0}; + }; + + static std::optional FindSymbolInScope(const std::string &name, const Scope &scope, Symbol::Type type); + + bool HasSymbol(const std::string &name) const; + bool HasSymbolLocalScope(const std::string &name) const; + + // @return true if it added a predefined identifier with that name + bool ConsumePredefinedIdentifier(const std::string &name); + + // Returns a freshly generated symbol. Previous mapping of the same name to a + // different symbol is replaced with the new one. + auto CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY, + int token_position = -1); + + auto GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY); + // Returns the symbol by name. If the mapping already exists, checks if the + // types match. Otherwise, returns a new symbol. + auto GetOrCreateSymbolLocalScope(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY); + + void VisitReturnBody(ReturnBody &body, Where *where = nullptr); + + void VisitWithIdentifiers(Expression *, const std::vector &); + + SymbolTable *symbol_table_; + + // Identifiers which are injected from outside the query. Each identifier + // is mapped by its name. + std::unordered_map predefined_identifiers_; + std::vector scopes_; + std::unordered_set prev_return_names_; + std::unordered_set curr_return_names_; +}; + +inline SymbolTable MakeSymbolTable(CypherQuery *query, const std::vector &predefined_identifiers = {}) { + SymbolTable symbol_table; + SymbolGenerator symbol_generator(&symbol_table, predefined_identifiers); + query->single_query_->Accept(symbol_generator); + for (auto *cypher_union : query->cypher_unions_) { + cypher_union->Accept(symbol_generator); + } + return symbol_table; +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/semantic/symbol_table.hpp b/src/query/v2/frontend/semantic/symbol_table.hpp new file mode 100644 index 000000000..a4ccf7e76 --- /dev/null +++ b/src/query/v2/frontend/semantic/symbol_table.hpp @@ -0,0 +1,64 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include + +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/semantic/symbol.hpp" +#include "utils/logging.hpp" + +namespace memgraph::query::v2 { + +class SymbolTable final { + public: + SymbolTable() {} + const Symbol &CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY, + int32_t token_position = -1) { + MG_ASSERT(table_.size() <= std::numeric_limits::max(), + "SymbolTable size doesn't fit into 32-bit integer!"); + auto got = table_.emplace(position_, Symbol(name, position_, user_declared, type, token_position)); + MG_ASSERT(got.second, "Duplicate symbol ID!"); + position_++; + return got.first->second; + } + + // TODO(buda): This is the same logic as in the cypher_main_visitor. During + // parsing phase symbol table doesn't exist. Figure out a better solution. + const Symbol &CreateAnonymousSymbol(Symbol::Type type = Symbol::Type::ANY) { + int id = 1; + while (true) { + static const std::string &kAnonPrefix = "anon"; + std::string name_candidate = kAnonPrefix + std::to_string(id++); + if (std::find_if(std::begin(table_), std::end(table_), [&name_candidate](const auto &item) -> bool { + return item.second.name_ == name_candidate; + }) == std::end(table_)) { + return CreateSymbol(name_candidate, false, type); + } + } + } + + const Symbol &at(const Identifier &ident) const { return table_.at(ident.symbol_pos_); } + const Symbol &at(const NamedExpression &nexpr) const { return table_.at(nexpr.symbol_pos_); } + const Symbol &at(const Aggregation &aggr) const { return table_.at(aggr.symbol_pos_); } + + // TODO: Remove these since members are public + int32_t max_position() const { return static_cast(table_.size()); } + + const auto &table() const { return table_; } + + int32_t position_{0}; + std::map table_; +}; + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/frontend/stripped.cpp b/src/query/v2/frontend/stripped.cpp new file mode 100644 index 000000000..3d50d57d2 --- /dev/null +++ b/src/query/v2/frontend/stripped.cpp @@ -0,0 +1,535 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/frontend/stripped.hpp" + +#include +#include +#include +#include +#include +#include + +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/opencypher/generated/MemgraphCypher.h" +#include "query/v2/frontend/opencypher/generated/MemgraphCypherBaseVisitor.h" +#include "query/v2/frontend/opencypher/generated/MemgraphCypherLexer.h" +#include "query/v2/frontend/parsing.hpp" +#include "query/v2/frontend/stripped_lexer_constants.hpp" +#include "utils/fnv.hpp" +#include "utils/logging.hpp" +#include "utils/string.hpp" + +namespace memgraph::query::v2::frontend { + +using namespace lexer_constants; + +StrippedQuery::StrippedQuery(const std::string &query) : original_(query) { + enum class Token { + UNMATCHED, + KEYWORD, // Including true, false and null. + SPECIAL, // +, .., +=, (, { and so on. + STRING, + INT, // Decimal, octal and hexadecimal. + REAL, + PARAMETER, + ESCAPED_NAME, + UNESCAPED_NAME, + SPACE + }; + + std::vector> tokens; + std::string unstripped_chunk; + for (int i = 0; i < static_cast(original_.size());) { + Token token = Token::UNMATCHED; + int len = 0; + auto update = [&](int new_len, Token new_token) { + if (new_len > len) { + len = new_len; + token = new_token; + } + }; + update(MatchKeyword(i), Token::KEYWORD); + update(MatchSpecial(i), Token::SPECIAL); + update(MatchString(i), Token::STRING); + update(MatchDecimalInt(i), Token::INT); + update(MatchOctalInt(i), Token::INT); + update(MatchHexadecimalInt(i), Token::INT); + update(MatchReal(i), Token::REAL); + update(MatchParameter(i), Token::PARAMETER); + update(MatchEscapedName(i), Token::ESCAPED_NAME); + update(MatchUnescapedName(i), Token::UNESCAPED_NAME); + update(MatchWhitespaceAndComments(i), Token::SPACE); + if (token == Token::UNMATCHED) throw LexingException("Invalid query."); + tokens.emplace_back(token, original_.substr(i, len)); + i += len; + + // If we notice execute, we possibly create a trigger which has defined statements. + // The statements will be parsed separately later on so we skip it for now. + if (utils::IEquals(tokens.back().second, "execute")) { + // check if it's CREATE TRIGGER query + std::span token_span{tokens}; + + // query could start with spaces and/or comments + if (token_span.front().first == Token::SPACE) { + token_span = token_span.subspan(1); + } + + // we need to check that first and third elements are correct keywords + // CREATETRIGGERtrigger-name...EXECUTE + // trigger-name (5th element) can also be "execute" so we verify that the size is larger than 5 + if (token_span.size() > 5 && utils::IEquals(token_span[0].second, "create") && + utils::IEquals(token_span[2].second, "trigger")) { + unstripped_chunk = original_.substr(i); + break; + } + } + } + + std::vector token_strings; + // A helper function that stores literal and its token position in a + // literals_. In stripped query text literal is replaced with a new_value. + // new_value can be any value that is lexed as a literal. + auto replace_stripped = [this, &token_strings](int position, const auto &value, const std::string &new_value) { + literals_.Add(position, storage::v3::PropertyValue(value)); + token_strings.push_back(new_value); + }; + + // Copy original tokens because we need to use original case in named + // expressions and keywords in tokens will be lowercased in the next loop. + auto original_tokens = tokens; + // For every token in original query remember token index in stripped query. + std::vector position_mapping(tokens.size(), -1); + + // Convert tokens to strings, perform filtering, store literals and nonaliased + // named expressions in return. + for (int i = 0; i < static_cast(tokens.size()); ++i) { + auto &token = tokens[i]; + + // We need to shift token index for every parameter since antlr's parser + // thinks of parameter as two tokens. + int token_index = token_strings.size() + parameters_.size(); + switch (token.first) { + case Token::UNMATCHED: + LOG_FATAL("Shouldn't happen"); + case Token::KEYWORD: { + // We don't strip NULL, since it can appear in special expressions + // like IS NULL and IS NOT NULL, but we strip true and false keywords. + if (utils::IEquals(token.second, "true")) { + replace_stripped(token_index, true, kStrippedBooleanToken); + } else if (utils::IEquals(token.second, "false")) { + replace_stripped(token_index, false, kStrippedBooleanToken); + } else { + token_strings.push_back(token.second); + } + } break; + case Token::SPACE: + break; + case Token::STRING: + replace_stripped(token_index, ParseStringLiteral(token.second), kStrippedStringToken); + break; + case Token::INT: + replace_stripped(token_index, ParseIntegerLiteral(token.second), kStrippedIntToken); + break; + case Token::REAL: + replace_stripped(token_index, ParseDoubleLiteral(token.second), kStrippedDoubleToken); + break; + case Token::SPECIAL: + case Token::ESCAPED_NAME: + case Token::UNESCAPED_NAME: + token_strings.push_back(token.second); + break; + case Token::PARAMETER: + parameters_[token_index] = ParseParameter(token.second); + token_strings.push_back(token.second); + break; + } + + if (token.first != Token::SPACE) { + position_mapping[i] = token_index; + } + } + + if (!unstripped_chunk.empty()) { + token_strings.push_back(std::move(unstripped_chunk)); + } + + query_ = utils::Join(token_strings, " "); + hash_ = utils::Fnv(query_); + + auto it = tokens.begin(); + while (it != tokens.end()) { + // Store nonaliased named expressions in returns in named_exprs_. + it = std::find_if(it, tokens.end(), + [](const std::pair &a) { return utils::IEquals(a.second, "return"); }); + // There is no RETURN so there is nothing to do here. + if (it == tokens.end()) return; + // Skip RETURN; + ++it; + + // Now we need to parse cypherReturn production from opencypher grammar. + // Skip leading whitespaces and DISTINCT statemant if there is one. + while (it != tokens.end() && it->first == Token::SPACE) { + ++it; + } + if (it != tokens.end() && utils::IEquals(it->second, "distinct")) { + ++it; + } + + // If the query is invalid, either antlr parser or cypher_main_visitor will + // report an error. + // TODO: we shouldn't rely on the fact that those checks will be done + // after this step. We should do them here. + while (it < tokens.end()) { + // Disregard leading whitespace + while (it != tokens.end() && it->first == Token::SPACE) { + ++it; + } + // There is only whitespace, nothing to do... + if (it == tokens.end()) break; + + bool has_as = false; + auto last_non_space = it; + auto jt = it; + // We should track number of opened braces and parantheses so that we can + // recognize if comma is a named expression separator or part of the + // list literal / function call. + int num_open_braces = 0; + int num_open_parantheses = 0; + int num_open_brackets = 0; + for (; + jt != tokens.end() && (jt->second != "," || num_open_braces || num_open_parantheses || num_open_brackets) && + !utils::IEquals(jt->second, "order") && !utils::IEquals(jt->second, "skip") && + !utils::IEquals(jt->second, "limit") && !utils::IEquals(jt->second, "union") && + !utils::IEquals(jt->second, "query") && jt->second != ";"; + ++jt) { + if (jt->second == "(") { + ++num_open_parantheses; + } else if (jt->second == ")") { + --num_open_parantheses; + } else if (jt->second == "[") { + ++num_open_braces; + } else if (jt->second == "]") { + --num_open_braces; + } else if (jt->second == "{") { + ++num_open_brackets; + } else if (jt->second == "}") { + --num_open_brackets; + } + has_as |= utils::IEquals(jt->second, "as"); + if (jt->first != Token::SPACE) { + last_non_space = jt; + } + } + if (!has_as) { + // Named expression is not aliased. Save string disregarding leading and + // trailing whitespaces. + std::string s; + auto begin_token = it - tokens.begin() + original_tokens.begin(); + auto end_token = last_non_space - tokens.begin() + original_tokens.begin() + 1; + for (auto kt = begin_token; kt != end_token; ++kt) { + s += kt->second; + } + named_exprs_[position_mapping[it - tokens.begin()]] = s; + } + if (jt != tokens.end() && jt->second == ",") { + // There are more named expressions. + it = jt + 1; + } else { + // We're done with this return statement + break; + } + } + } +} + +std::string GetFirstUtf8Symbol(const char *_s) { + // According to + // https://stackoverflow.com/questions/16260033/reinterpret-cast-between-char-and-stduint8-t-safe + // this checks if casting from const char * to uint8_t is undefined behaviour. + static_assert(std::is_same::value, + "This library requires std::uint8_t to be implemented as " + "unsigned char."); + const uint8_t *s = reinterpret_cast(_s); + if ((*s >> 7) == 0x00) return std::string(_s, _s + 1); + if ((*s >> 5) == 0x06) { + auto *s1 = s + 1; + if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); + return std::string(_s, _s + 2); + } + if ((*s >> 4) == 0x0e) { + auto *s1 = s + 1; + if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); + auto *s2 = s + 2; + if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); + return std::string(_s, _s + 3); + } + if ((*s >> 3) == 0x1e) { + auto *s1 = s + 1; + if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); + auto *s2 = s + 2; + if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); + auto *s3 = s + 3; + if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character."); + return std::string(_s, _s + 4); + } + throw LexingException("Invalid character."); +} + +// Return codepoint of first utf8 symbol and its encoded length. +std::pair GetFirstUtf8SymbolCodepoint(const char *_s) { + static_assert(std::is_same::value, + "This library requires std::uint8_t to be implemented as " + "unsigned char."); + const uint8_t *s = reinterpret_cast(_s); + if ((*s >> 7) == 0x00) return {*s & 0x7f, 1}; + if ((*s >> 5) == 0x06) { + auto *s1 = s + 1; + if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); + return {((*s & 0x1f) << 6) | (*s1 & 0x3f), 2}; + } + if ((*s >> 4) == 0x0e) { + auto *s1 = s + 1; + if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); + auto *s2 = s + 2; + if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); + return {((*s & 0x0f) << 12) | ((*s1 & 0x3f) << 6) | (*s2 & 0x3f), 3}; + } + if ((*s >> 3) == 0x1e) { + auto *s1 = s + 1; + if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character."); + auto *s2 = s + 2; + if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character."); + auto *s3 = s + 3; + if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character."); + return {((*s & 0x07) << 18) | ((*s1 & 0x3f) << 12) | ((*s2 & 0x3f) << 6) | (*s3 & 0x3f), 4}; + } + throw LexingException("Invalid character."); +} + +// From here until end of file there are functions that calculate matches for +// every possible token. Functions are more or less compatible with Cypher.g4 +// grammar. Unfortunately, they contain a lof of special cases and shouldn't +// be changed without good reasons. +// +// Here be dragons, do not touch! +// ____ __ +// { --.\ | .)%%%)%% +// '-._\\ | (\___ %)%%(%%(%%% +// `\\|{/ ^ _)-%(%%%%)%%;%%% +// .'^^^^^^^ /` %%)%%%%)%%%' +// //\ ) , / '%%%%(%%' +// , _.'/ `\<-- \< +// `^^^` ^^ ^^ +int StrippedQuery::MatchKeyword(int start) const { return kKeywords.Match(original_.c_str() + start); } + +int StrippedQuery::MatchSpecial(int start) const { return kSpecialTokens.Match(original_.c_str() + start); } + +int StrippedQuery::MatchString(int start) const { + if (original_[start] != '"' && original_[start] != '\'') return 0; + char start_char = original_[start]; + for (auto *p = original_.data() + start + 1; *p; ++p) { + if (*p == start_char) return p - (original_.data() + start) + 1; + if (*p == '\\') { + ++p; + if (*p == '\\' || *p == '\'' || *p == '"' || *p == 'B' || *p == 'b' || *p == 'F' || *p == 'f' || *p == 'N' || + *p == 'n' || *p == 'R' || *p == 'r' || *p == 'T' || *p == 't') { + // Allowed escaped characters. + continue; + } else if (*p == 'U' || *p == 'u') { + int cnt = 0; + auto *r = p + 1; + while (isxdigit(*r) && cnt < 8) { + ++cnt; + ++r; + } + if (!*r) return 0; + if (cnt < 4) return 0; + if (cnt >= 4 && cnt < 8) { + p += 4; + } + if (cnt >= 8) { + p += 8; + } + } else { + return 0; + } + } + } + return 0; +} + +int StrippedQuery::MatchDecimalInt(int start) const { + if (original_[start] == '0') return 1; + int i = start; + while (i < static_cast(original_.size()) && isdigit(original_[i])) { + ++i; + } + return i - start; +} + +int StrippedQuery::MatchOctalInt(int start) const { + if (original_[start] != '0') return 0; + int i = start + 1; + while (i < static_cast(original_.size()) && '0' <= original_[i] && original_[i] <= '7') { + ++i; + } + if (i == start + 1) return 0; + return i - start; +} + +int StrippedQuery::MatchHexadecimalInt(int start) const { + if (original_[start] != '0') return 0; + if (start + 1 >= static_cast(original_.size())) return 0; + if (original_[start + 1] != 'x') return 0; + int i = start + 2; + while (i < static_cast(original_.size()) && isxdigit(original_[i])) { + ++i; + } + if (i == start + 2) return 0; + return i - start; +} + +int StrippedQuery::MatchReal(int start) const { + enum class State { START, BEFORE_DOT, DOT, AFTER_DOT, E, E_MINUS, AFTER_E }; + State state = State::START; + auto i = start; + while (i < static_cast(original_.size())) { + if (original_[i] == '.') { + if (state != State::BEFORE_DOT && state != State::START) break; + state = State::DOT; + } else if ('0' <= original_[i] && original_[i] <= '9') { + if (state == State::START) { + state = State::BEFORE_DOT; + } else if (state == State::DOT) { + state = State::AFTER_DOT; + } else if (state == State::E || state == State::E_MINUS) { + state = State::AFTER_E; + } + } else if (original_[i] == 'e' || original_[i] == 'E') { + if (state != State::BEFORE_DOT && state != State::AFTER_DOT) break; + state = State::E; + } else if (original_[i] == '-') { + if (state != State::E) break; + state = State::E_MINUS; + } else { + break; + } + ++i; + } + if (state == State::DOT) --i; + if (state == State::E) --i; + if (state == State::E_MINUS) i -= 2; + return i - start; +} + +int StrippedQuery::MatchParameter(int start) const { + int len = original_.size(); + if (start + 1 == len) return 0; + if (original_[start] != '$') return 0; + int max_len = 0; + max_len = std::max(max_len, MatchUnescapedName(start + 1)); + max_len = std::max(max_len, MatchEscapedName(start + 1)); + max_len = std::max(max_len, MatchKeyword(start + 1)); + max_len = std::max(max_len, MatchDecimalInt(start + 1)); + if (max_len == 0) return 0; + return 1 + max_len; +} + +int StrippedQuery::MatchEscapedName(int start) const { + int len = original_.size(); + int i = start; + while (i < len) { + if (original_[i] != '`') break; + int j = i + 1; + while (j < len && original_[j] != '`') { + ++j; + } + if (j == len) break; + i = j + 1; + } + return i - start; +} + +int StrippedQuery::MatchUnescapedName(int start) const { + auto i = start; + auto got = GetFirstUtf8SymbolCodepoint(original_.data() + i); + if (got.first >= lexer_constants::kBitsetSize || !kUnescapedNameAllowedStarts[got.first]) { + return 0; + } + i += got.second; + while (i < static_cast(original_.size())) { + got = GetFirstUtf8SymbolCodepoint(original_.data() + i); + if (got.first >= lexer_constants::kBitsetSize || !kUnescapedNameAllowedParts[got.first]) { + break; + } + i += got.second; + } + return i - start; +} + +int StrippedQuery::MatchWhitespaceAndComments(int start) const { + enum class State { OUT, IN_LINE_COMMENT, IN_BLOCK_COMMENT }; + State state = State::OUT; + int i = start; + int len = original_.size(); + // We need to remember at which position comment started because if we fail + // to match comment finish we have a match until comment start position. + int comment_position = -1; + while (i < len) { + if (state == State::OUT) { + auto got = GetFirstUtf8SymbolCodepoint(original_.data() + i); + if (got.first < lexer_constants::kBitsetSize && kSpaceParts[got.first]) { + i += got.second; + } else if (i + 1 < len && original_[i] == '/' && original_[i + 1] == '*') { + comment_position = i; + state = State::IN_BLOCK_COMMENT; + i += 2; + } else if (i + 1 < len && original_[i] == '/' && original_[i + 1] == '/') { + comment_position = i; + if (i + 2 < len) { + // Special case for an empty line comment starting right at the end of + // the query. + state = State::IN_LINE_COMMENT; + } + i += 2; + } else { + break; + } + } else if (state == State::IN_LINE_COMMENT) { + if (original_[i] == '\n') { + state = State::OUT; + ++i; + } else if (i + 1 < len && original_[i] == '\r' && original_[i + 1] == '\n') { + state = State::OUT; + i += 2; + } else if (original_[i] == '\r') { + break; + } else if (i + 1 == len) { + state = State::OUT; + ++i; + } else { + ++i; + } + } else if (state == State::IN_BLOCK_COMMENT) { + if (i + 1 < len && original_[i] == '*' && original_[i + 1] == '/') { + i += 2; + state = State::OUT; + } else { + ++i; + } + } + } + if (state != State::OUT) return comment_position - start; + return i - start; +} + +} // namespace memgraph::query::v2::frontend diff --git a/src/query/v2/frontend/stripped.hpp b/src/query/v2/frontend/stripped.hpp new file mode 100644 index 000000000..e70a9c671 --- /dev/null +++ b/src/query/v2/frontend/stripped.hpp @@ -0,0 +1,103 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include + +#include "query/v2/parameters.hpp" +#include "utils/fnv.hpp" + +namespace memgraph::query::v2::frontend { + +// Strings used to replace original tokens. Different types are replaced with +// different token. +const std::string kStrippedIntToken = "0"; +const std::string kStrippedDoubleToken = "0.0"; +const std::string kStrippedStringToken = "\"a\""; +const std::string kStrippedBooleanToken = "true"; + +/** + * StrippedQuery contains: + * * stripped query + * * literals stripped from query + * * hash of stripped query + */ +class StrippedQuery { + public: + /** + * Strips the input query and stores stripped query, stripped arguments and + * stripped query hash. + * + * @param query Input query. + */ + explicit StrippedQuery(const std::string &query); + + /** + * Copy constructor is deleted because we don't want to make unnecessary + * copies of this object (copying of string and vector could be expensive) + */ + StrippedQuery(const StrippedQuery &other) = delete; + StrippedQuery &operator=(const StrippedQuery &other) = delete; + + /** + * Move is allowed operation because it is not expensive and we can + * move the object after it was created. + */ + StrippedQuery(StrippedQuery &&other) = default; + StrippedQuery &operator=(StrippedQuery &&other) = default; + + const std::string &query() const { return query_; } + const auto &original_query() const { return original_; } + const auto &literals() const { return literals_; } + const auto &named_expressions() const { return named_exprs_; } + const auto ¶meters() const { return parameters_; } + uint64_t hash() const { return hash_; } + + private: + // Return len of matched keyword if something is matched, otherwise 0. + int MatchKeyword(int start) const; + int MatchString(int start) const; + int MatchSpecial(int start) const; + int MatchDecimalInt(int start) const; + int MatchOctalInt(int start) const; + int MatchHexadecimalInt(int start) const; + int MatchReal(int start) const; + int MatchParameter(int start) const; + int MatchEscapedName(int start) const; + int MatchUnescapedName(int start) const; + int MatchWhitespaceAndComments(int start) const; + + // Original query. + std::string original_; + + // Stripped query. + std::string query_; + + // Token positions of stripped out literals mapped to their values. + // TODO: Parameters class really doesn't provide anything interesting. This + // could be changed to std::unordered_map, but first we need to rewrite (or + // get rid of) hardcoded queries which expect Parameters. + Parameters literals_; + + // Token positions of query parameters mapped to their names. + std::unordered_map parameters_; + + // Token positions of nonaliased named expressions in return statement mapped + // to their original (unstripped) string. + std::unordered_map named_exprs_; + + // Hash based on the stripped query. + uint64_t hash_; +}; + +} // namespace memgraph::query::v2::frontend diff --git a/src/query/v2/frontend/stripped_lexer_constants.hpp b/src/query/v2/frontend/stripped_lexer_constants.hpp new file mode 100644 index 000000000..4e52fbdc4 --- /dev/null +++ b/src/query/v2/frontend/stripped_lexer_constants.hpp @@ -0,0 +1,2924 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include +#include +#include + +namespace memgraph::query::v2 { +namespace lexer_constants { + +namespace trie { + +// Trie data structure implemented to be used in StrippedQuery. If you want to +// change it please rerun benchmark/stripped to be sure that performance of +// StrippedQuery is not degraded by the change. Also there are no tests that +// directly test this class, but there are tests that test StrippedQuery. + +namespace detail { +inline int Noop(int x) { return x; } +}; // namespace detail + +class Trie { + public: + Trie() {} + Trie(std::initializer_list l) { + for (const auto &s : l) { + Insert(s); + } + } + + void Insert(const std::string &s) { + int node_id = kRootIndex; + for (const auto &_c : s) { + const unsigned char &c = reinterpret_cast(_c); + int &next_node_id = nodes_[node_id].next[c]; + if (next_node_id == 0) { + next_node_id = nodes_.size(); + // First assign then emplace_back because after emplace_back reference + // could be invalid. + node_id = next_node_id; + nodes_.emplace_back(); + } else { + node_id = next_node_id; + } + } + nodes_[node_id].finish = true; + } + + template + int Match(const char *s) const { + int node_id = kRootIndex; + int longest_found_len = 0; + int i = 1; + for (const char *p = s; *p; ++p, ++i) { + const unsigned char &c = reinterpret_cast(*p); + node_id = nodes_[node_id].next[Map(c)]; + if (node_id == 0) break; + if (nodes_[node_id].finish) { + longest_found_len = i; + } + } + return longest_found_len; + } + + private: + struct Node { + int next[1 << (sizeof(unsigned char) * 8)] = {}; + bool finish = false; + }; + + const static int kRootIndex = 0; + std::vector nodes_{1}; +}; +} // namespace trie + +// All word constants should be lowercase in this file. + +const int kBitsetSize = 65536; + +const trie::Trie kKeywords = {"union", + "all", + "optional", + "match", + "unwind", + "as", + "merge", + "on", + "create", + "set", + "detach", + "delete", + "remove", + "with", + "distinct", + "return", + "order", + "by", + "skip", + "limit", + "ascending", + "asc", + "descending", + "desc", + "where", + "or", + "xor", + "and", + "not", + "in", + "starts", + "ends", + "contains", + "is", + "null", + "case", + "when", + "then", + "else", + "end", + "count", + "filter", + "extract", + "any", + "none", + "single", + "true", + "false", + "reduce", + "coalesce", + "user", + "password", + "alter", + "drop", + "show", + "stats", + "unique", + "explain", + "profile", + "storage", + "index", + "info", + "exists", + "assert", + "constraint", + "node", + "key", + "dump", + "database", + "call", + "yield", + "memory", + "mb", + "kb", + "unlimited", + "free", + "procedure", + "query", + "free_memory", + "read_file", + "lock_path", + "after", + "before", + "execute", + "transaction", + "trigger", + "triggers", + "update", + "comitted", + "uncomitted", + "global", + "isolation", + "level", + "next", + "read", + "session", + "snapshot", + "transaction", + "batch_limit", + "batch_interval", + "batch_size", + "consumer_group", + "start", + "stream", + "streams", + "transform", + "topics", + "check", + "setting", + "settings", + "bootstrap_servers", + "kafka", + "pulsar", + "service_url", + "version", + "websocket" + "foreach"}; + +// Unicode codepoints that are allowed at the start of the unescaped name. +const std::bitset kUnescapedNameAllowedStarts( + std::string("00000000000000000000000000000000000111001111110011111100111111000111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111100000000000111111111111111111111111110100001111111111111111111111111" + "10000000000000000000000000000000000001111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111101111100000000000000000000000000000000111000000000" + "00000000000000011000000000000000000000000000000000000000000000000000000011" + "11111111110000000000000000000000000000000000000000111111111111111111111111" + "11111111111111111111111111111100111111111111111111111111111111111111111111" + "11111111111111111111110000000000000000001111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111100000000000000000000000000000000011111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111011011010111110111111111111101111111111010000011" + "11100000000000011111110000000000000000000000000000000000000011111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111100111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111110000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000111111111111111111111111111111111111111111" + "11111110000111111111111111111111110000000000001111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111000000000000000000000000000001" + "11111111111111111111111111111111110000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000001111111011111110000000001111110011111100111" + "11100000000000011100000001111111111100111000000000000000000000000101001111" + "10011000101111111111111111111111111111111111111111111111110000010001111111" + "11111111111111110000000000000000000011111111011100000000000000000000000111" + "11111111111111111111111111111111111111000000000000000000000000000000000000" + "00000000000010000000000000000000000000000111111111111111111111111111111111" + "11111111111111000000011111111111111111111111111111000000000000000000000000" + "01111111111111111111111100000000001111111111111111111111111111000000000000" + "00100011111100000000000000000000000000000000000000000000000000000000000000" + "11111111111111111111111111111111111111111111111111000000000000001111111111" + "11111111111111111111111111111111111111111100000000000000000000000000000111" + "11111111111111111111011110111011111111110000000000000000000000000000000000" + "00000000000000000000000000000000000000000001111111111100000000000011110111" + "10011111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111001111111110000000000000000000000000000000" + "00000000111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111000000001111111111111111111111111000000000000000011111111111" + "11111111111111111111111111111111111100000000000000000000110000000000111111" + "11111111110001111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111100111111111111" + "11111111111111111111111111111111110000000000000000000000000000000000000000" + "00000000000000000000000000011111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111000000000000000000000000000000000000000000000000000111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111110000000000000000" + "00000000000000000000000000000000000000000000000000000000001111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111100000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000011111111111111110000000000" + "00000000000000000000000000000000000000000001111111111111111111111111110000" + "00000000000001111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111100011111111111111111111111111111111111111" + "11100000111101111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111011111000011111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111100001111100111110000000" + "11111111100000000000000000000000001110000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000011111" + "11011111110111111101111111011111110111111101111111011111110000000001111111" + "11111111111111110000000000000000100000001111111111111111111111111111111111" + "11111111111111111111110010000010111111111111111111111111111111111111110000" + "00000000110001111000000111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111011111111111111111111111111111111111111111111111011111111111111111" + "11111111111111111111111111111100000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000011111111111111111111111111111" + "11111111111100000000000000000100001111100000111100111111111111111101010100" + "00001111110010111111111100100001000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000111111111111100" + "00000000000000100000000000001000000000000000000000000000010000000000000000" + "00011000000000000000000000000000000000000000000000000000000000000000000111" + "11110111000001111111111111000011111100111100011111110111000101111111011111" + "11111111111111111111111111111111111111111111111100111111111111111111111111" + "11111110101010111111110011111100111111111111111111111111111111111111110011" + "11110011111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111000000000000" + "00000000000000000000000000000000000000000000000000001111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111110000000001100011110111100000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00001111111111111111111111111111111111110000000000111000000000000000000000" + "00000000000000000000111111111111111111111111111111111111000000000000000000" + "00000000111111111111111111111111111111111111111111110000000000110000000000" + "00011111111111111111111111111111100000000000000000000000000000000000000000" + "00000000000000111111100000000000000000111111111111111111111111111111111111" + "11111111111000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000010000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000011111111111111111111111111111111111" + "11111111111111111100000000011111111111111111111111000000000000000000000000" + "00000000000000000000000000000000111111100000000000000000000011111111111111" + "11111111111111111111111111111100000000000111110011111111111111111111111111" + "11110000000000000000000000000000000000000000000000000001111111111111111111" + "11111111110000000000111111111111111111111111111111111111111111111111111111" + "11111111111111110000010111111111111111111111111111111111111111110000000011" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111100000000000000000000000000000000000000000000000000000000000000" + "00000100001000000000000000000000000000000000001111111111111111111111111111" + "11111111111111111111111100000000000000011101111111111111000000000000001111" + "11111111111111000000000000001111111111111111110000000000000011110111111111" + "11110000000000000001110001111111111111111111111111111111111111111111111111" + "11111111111111111111111111000001111111111111111111111111101111111111111111" + "10011111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111110000000000001111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111100000000000000001111" + "11111111111100000000000000000000000000000000000001111111111111111111111111" + "11111111111111111111111111111111111111111100111101111111111111111111111111" + "11111111111111111111111111111111011111111111111100111101011111110011110111" + "11111111111111111111111111111100111101111111111111111111111111111111111111" + "11110011110101111111001111011111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111110111111111" + "11111111111111111111111111111111110010000010111111111111111111111111111111" + "11111111000000000000000001000000000000111111111111100001110000000110001000" + "11110000111111000000000000000010000000000000000000011111111111111111111111" + "11111111111111111111000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000001111100000000" + "00000000000000000001111111111111111111111111111111111110111111110000000000" + "00000000000000000000000000000000000000000000000000000100000000000000000000" + "00000000000011110000000000000000000001011111001000000000110111101100101011" + "10111111101111000000100101100101100000000000000000000000000000000000000000" + "00000000000000000111111100000000000011011111111111111111111111111111111111" + "11111111111110000000000000000000000000000000000000000000000000000000000111" + "11110010111111111011111111111111111111111100011111111111111111100000111111" + "00000000000000000000000011000000000000000001000000000000000010011111111111" + "11111111111111111111111111111101110111111110000000000000000001100000000000" + "00001101000000000000000000000000000000001000111110111111111101111111111111" + "11111111110111011111111000000000000000000000000000000000001100000011000000" + "00000000000000000000100011111011111111110111111111111111111111110111011111" + "11100000000000000000000000000000000000000000000000000001000000000000000000" + "00001111111111110001110001100011010110001111011100011111101000000000000000" + "00100000000000000011101100000000000000000000000000000010001111101101111111" + "01111111111111111111111001100111111110000000000000000000000000000000000011" + "00000000000000010000000000000000001000111110110111111101111111111111111111" + "11101110111111111000000000000000011100000000000000000001011110000000000000" + "00000000000000000011011011011111110111111111111111111111100110000111111000" + "00000000000000001100000000000000111011000000000000010000000000000000100011" + "11000101111111011111111111111111111110011001111111100000111111101111111000" + "00000000000011111111110000000100000000000000000010001111111111111111111111" + "11111111111111111111111111111111000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000011111111111010000000000000000" + "00000000000000000000000000000000000000000000000000000001111111111111111111" + "11111100000000000000000000000100010000000001000011111111111111111111110000" + "01000011000000000111111111111111111111111111111111000000000000000000000000" + "10000000000011111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111100000000000000000000000000000111111111111111111" + "11111111111101000000000000000010011100000000001100000001100000000000000010" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111110110000000000000000000000000000000000011111111111" + "11111111111111111111111111111111000000000000000000000000000000000000000000" + "00011100000111111111111111111111111111000000000000000000000000000000000000" + "00000000000000000000000000000000000011111111111111111111111111111111111111" + "10000000100111111111111111111111111111111111111110000000001111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111000000" + "00111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111110111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11101111111111111111111101011101000000001111001101111100000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000101000000011111000000000000001111111111" + "11000011111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111011111111111111111111111111111110111111111111111111111" + "11000001000010000000000100000000000000000000000000000000000000000000000111" + "11111111111111111111111010000111111111111111111111111110000000000000000000" + "0000000000000000000000000000000000000000000000")); + +// Unicode codepoints that are allowed at the middle of the unescaped name. +const std::bitset kUnescapedNameAllowedParts( + std::string("00000000000000000000000001100011000111001111110011111100111111000111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111100000000000111111111111111111111111110100001111111111111111111111111" + "10000000111111111100000000000100000001111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111101111100000010000000000000000000000000111000000000" + "00000000000000011000000000000111111100000000000000001111111111111111000111" + "11111111110000000000000000000000000000000000000000111111111111111111111111" + "11111111111111111111111111111100111111111111111111111111111111111111111111" + "11111111111111111111110000000000000000001111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111100000000000000000000000000000000011111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111011011010111110111111111111101111111111110000011" + "11100000000000011111110000000000000000000000000000000000000011111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111100111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111110000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000111111111111111111111111111111111111111111" + "11111110000111111111111111111111110000000000001111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111000000111111111100110111111111" + "11111111111111111111111111111111110000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000001111111011111110000000001111110011111100111" + "11100000000001111100111111111111111100111000000000000000000000000111111111" + "11111111111111111111111111111111111111111111111111111111110000110001111111" + "11111111111111110000001111111111001111111111111100000000011111111111111111" + "11111111111111111111111111111111111111000000000000000000000000000000000000" + "00111111111110000000000000011111111111111111111111111111111111111111111111" + "11111111111111111100011111111111111111111111111111000000000000111111111111" + "11111111111111111111111100111111111111111111111111111111111111111111111100" + "00100011111111111111111111111100000011111111110000000000011111111111111111" + "11111111111111111111111111111111111111111111111111110000000000001111111111" + "11111111111111111111111111111111111111111100000001000000000000000011111111" + "11111111111111111111111111111111111111110000000000000000000000000000000000" + "00000000000000000000000000000000000000000001111111111100000000000011110111" + "10011111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111001111111110000000000000000000000000000000" + "00000011111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111100000001111111111111111111111111011111111110000111111111111" + "11111111111111111111111111111111111100000000000000000000111111111111111111" + "11111111110001111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111100111111111111" + "11111111111111111111111111111111110000000000000000000000000000000000000000" + "00000000000000000000000000011111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111000000000000000000000000000000000000000000000000000111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111110000000000000000" + "00000000000000000000000000000000000000000000000000000000001111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111100000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000011111111111111110000000000" + "00000000000000000000000000000000000000000001111111111111111111111111110000" + "00000000000001111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111100011111111111111111111111111111111111111" + "11100000111101111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111011111110011111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111100001111100111110111111" + "11111111100000000000000000000000001110000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000011111111111111111111111111111111011111" + "11011111110111111101111111011111110111111101111111011111110000000001111111" + "11111111111111111000000000000000100000001111111111111111111111111111111111" + "11111111111111111111110010000010111111111111111111111111111111111111110000" + "00000000111111111000000111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111011111111111111111111111111111111111111111111111011111111111111111" + "11111111111111111111111111111100000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000011111111111111111111111111111" + "11111111111100000000000000000100001111100000111100111111111111111101010100" + "00001111110010111111111100100001000000000000000001111111111110001000011111" + "11111111000000000000000000000111111111111111111111111111000111111111111100" + "00000000000000100000000000001000000000000000000000000000010000000000000000" + "00011000000000000000000000000000000000000000000000000000000000000000000111" + "11110111000001111111111111000011111100111100011111110111000101111111011111" + "11111111111111111111111111111111111111111111111100111111111111111111111111" + "11111110101010111111110011111100111111111111111111111111111111111111110011" + "11110011111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111100000000" + "00000000000001111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111110000000001111111111111111111111111111111111101110000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00001111111111111111111111111111111111111111111111111000111111111100000000" + "11111111111111111111111111111111111111111111111111111111000000000000111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111100000000000011111111100000000000000000" + "11111111110000111111111111111111111111111111111111111111111111111111111111" + "11111111111111110000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000010000000000000111111111100000011111111111001" + "11111111111111111111111111110111111111111111111111111111111111111111111111" + "11111111111111111100001111111111111111111111111111000000000000000000000000" + "00000000000001111111111100000011111111111111111111111111000011111111111111" + "11111111111111111111111111111100000000000111110011111111111111111111111111" + "11111111111111000000000011111111111100001111111111110001111111111111111111" + "11111111110000000000111111111111111111111111111111111111111111111111111111" + "11111111111111110000011111111111111111111111111111111111111111110000000011" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111100000011111111110011100000000000000000000000000000000011111111" + "11001110001000111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111100000000000011011101111111111111000000000000111111" + "11111111111111000000000001111111111111111111110000000000011111110111111111" + "11110000000000000001110001111111111111111111111111111111111111111111111111" + "11111111111111111111111111000001111111111111111111111111101111111111111111" + "10011111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111110000000000001111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111100000000000000001111" + "11111111111100000000000000111111111000000000111001111111111111111111111111" + "11111111111111111111111111111111111111111100111101111111111111111111111111" + "11111111111111111111111111111111011111111111111100111101011111110011110111" + "11111111111111111111111111111100111101111111111111111111111111111111111111" + "11110011110101111111001111011111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111110111111111" + "11111111111111111111111111111111110010000010111111111111111111111111111111" + "11111111001111111111111111111111111111111111111111111111111111111111111111" + "11111111111111000000111111111111111111111111111111111111111111111111111111" + "11111111111111111111000000000000000000000000000000000000000000000000000000" + "00010000000001111111111111111111111111111111111110111111111111111111011111" + "11111111111111100001111111111111111111111111111111111110111111111100001010" + "10000000000011111111110000001100000000000000000000000100000000000000000000" + "00000000000011110011111111110011111101011111001110111111111111101100101011" + "10111111101111000000100101100101100000000000000000000000000000000000000011" + "11111111011111111111111110000111111111111111111111111111111111111111111111" + "11111111111110000000000000110000000000000000001111111101011111100001000111" + "11110010111111111011111111111111111111111100011111111111111111101100111111" + "00000000001111111111001111000000001000000001111101110111111110011111111111" + "11111111111111111111111111111101110111111110110000000000000001101111111111" + "00111101000000011000000011110111011111111100111110111111111101111111111111" + "11111111110111011111111011000000000000000000111111111100111100000011011000" + "00001111011101111111100011111011111111110111111111111111111111110111011111" + "11101110000000100000000011111111110000000000000010000001001111011100011111" + "00001111111111110001110001100011010110001111011100011111101100000000000000" + "00101111111111001111101100001100000000111001100111111111001111101101111111" + "01111111111111111111111001100111111110111000000000000000101111111111001111" + "00000000000000010011101110111111111100111110110111111101111111111111111111" + "11101110111111111011100000000000111111111111111100000001011110000000100011" + "10011000011111010011011011011111110111111111111111111111100110000111111011" + "10000010000000111111111111110011111011000010000000011110011001111111110011" + "11000101111111011111111111111111111110011001111111101110111111101111111011" + "11111111001111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111101111111111111111111111111110000000000" + "00000000000000000000000000000000000000000000011111111111010000000000000000" + "00000000000000000000000000000000000000000000000000001111111111111111111111" + "11111100000000000000000011111111111111111111111111111111111111111111110000" + "01000011111111111111111111111111111111111111111111111111111100000000000000" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111100111111111111111111111111111111111111111111111" + "11111111111111000000000000000010011111111111111111110111111111100111111110" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111110000111111111111111111111111111111111111111111" + "11111111111111111111111111111111000001111111111100001000000000000000000000" + "00011100000111111111111111111111111111000000001011011010111111111111111111" + "11111111111111111111111111101000000011111111111111111111111111111111111111" + "10000000100111111111111111111111111111111111111110000000001111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111001111" + "10111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111110111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11101111111111111111111101011111000000001111001101111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111100000000000000000101000000011111000000000000001111111111" + "11000011111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111111111111111111111111111111111111111111111111111111111" + "11111111111111111111011111111111111111111111111111110111111111111111111111" + "11000001001010000000000100001111000000000000000000000000000000000000000111" + "11111111111111111111111010000111111111111111111111111110000000111111111100" + "0000000000000000000000000000000000000000000000")); + +const std::bitset kSpaceParts( + std::string("00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000100000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000001000000000000000000000000000" + "00000000000000000000100000110000000000000000000000000000011111111111000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000100000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000010000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000010000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000111110000000000000011111000000000")); + +const trie::Trie kSpecialTokens = {";", + ",", + "=", + "+=", + "*", + "(", + ")", + "[", + "]", + ":", + "|", + "..", + "+", + "-", + "/", + "%", + "^", + "=~", + "<>", + "!=", + "<", + ">", + "<=", + ">=", + ".", + "{", + "}", + "$", + "\xE2\x9F\xA8", // u8"\u27e8" + "\xE3\x80\x88", // u8"\u3008" + "\xEF\xB9\xA4", // u8"\ufe64" + "\xEF\xBC\x9C", // u8"\uff1c" + "\xE2\x9F\xA9", // u8"\u27e9" + "\xE3\x80\x89", // u8"\u3009" + "\xEF\xB9\xA5", // u8"\ufe65" + "\xEF\xBC\x9E", // u8"\uff1e" + "\xC2\xAD", // u8"\u00ad" + "\xE2\x80\x90", // u8"\u2010" + "\xE2\x80\x91", // u8"\u2011" + "\xE2\x80\x92", // u8"\u2012" + "\xE2\x80\x93", // u8"\u2013" + "\xE2\x80\x94", // u8"\u2014" + "\xE2\x80\x95", // u8"\u2015" + "\xE2\x88\x92", // u8"\u2212" + "\xEF\xB9\x98", // u8"\ufe58" + "\xEF\xB9\xA3", // u8"\ufe63" + "\xEF\xBC\x8D"}; // u8"\uff0d" +} // namespace lexer_constants +} // namespace memgraph::query::v2 diff --git a/src/query/v2/interpret/awesome_memgraph_functions.cpp b/src/query/v2/interpret/awesome_memgraph_functions.cpp new file mode 100644 index 000000000..9dbe3ccba --- /dev/null +++ b/src/query/v2/interpret/awesome_memgraph_functions.cpp @@ -0,0 +1,1323 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/interpret/awesome_memgraph_functions.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "query/v2/db_accessor.hpp" +#include "query/v2/exceptions.hpp" +#include "query/v2/procedure/cypher_types.hpp" +#include "query/v2/procedure/mg_procedure_impl.hpp" +#include "query/v2/procedure/module.hpp" +#include "query/v2/typed_value.hpp" +#include "utils/string.hpp" +#include "utils/temporal.hpp" + +namespace memgraph::query::v2 { +namespace { + +//////////////////////////////////////////////////////////////////////////////// +// eDSL using template magic for describing a type of an awesome memgraph +// function and checking if the passed in arguments match the description. +// +// To use the type checking eDSL, you should put a `FType` invocation in the +// body of your awesome Memgraph function. `FType` takes type arguments as the +// description of the function type signature. Each runtime argument will be +// checked in order corresponding to given compile time type arguments. These +// type arguments can come in two forms: +// +// * final, primitive type descriptor and +// * combinator type descriptor. +// +// The primitive type descriptors are defined as empty structs, they are right +// below this documentation. +// +// Combinator type descriptors are defined as structs taking additional type +// parameters, you can find these further below in the implementation. Of +// primary interest are `Or` and `Optional` type combinators. +// +// With `Or` you can describe that an argument can be any of the types listed in +// `Or`. For example, `Or` allows an argument to be either +// `Null` or a boolean or an integer. +// +// The `Optional` combinator is used to define optional arguments to a function. +// These must come as the last positional arguments. Naturally, you can use `Or` +// inside `Optional`. So for example, `Optional, Integer>` +// describes that a function takes 2 optional arguments. The 1st one must be +// either a `Null` or a boolean, while the 2nd one must be an integer. The type +// signature check will succeed in the following cases. +// +// * No optional arguments were supplied. +// * One argument was supplied and it passes `Or` check. +// * Two arguments were supplied, the 1st one passes `Or` check +// and the 2nd one passes `Integer` check. +// +// Runtime arguments to `FType` are: function name, pointer to arguments and the +// number of received arguments. +// +// Full example. +// +// FType, NonNegativeInteger, +// Optional>("substring", args, nargs); +// +// The above will check that `substring` function received the 2 required +// arguments. Optionally, the function may take a 3rd argument. The 1st argument +// must be either a `Null` or a character string. The 2nd argument is required +// to be a non-negative integer. If the 3rd argument was supplied, it will also +// be checked that it is a non-negative integer. If any of these checks fail, +// `FType` will throw a `QueryRuntimeException` with an appropriate error +// message. +//////////////////////////////////////////////////////////////////////////////// + +struct Null {}; +struct Bool {}; +struct Integer {}; +struct PositiveInteger {}; +struct NonZeroInteger {}; +struct NonNegativeInteger {}; +struct Double {}; +struct Number {}; +struct List {}; +struct String {}; +struct Map {}; +struct Edge {}; +struct Vertex {}; +struct Path {}; +struct Date {}; +struct LocalTime {}; +struct LocalDateTime {}; +struct Duration {}; + +template +bool ArgIsType(const TypedValue &arg) { + if constexpr (std::is_same_v) { + return arg.IsNull(); + } else if constexpr (std::is_same_v) { + return arg.IsBool(); + } else if constexpr (std::is_same_v) { + return arg.IsInt(); + } else if constexpr (std::is_same_v) { + return arg.IsInt() && arg.ValueInt() > 0; + } else if constexpr (std::is_same_v) { + return arg.IsInt() && arg.ValueInt() != 0; + } else if constexpr (std::is_same_v) { + return arg.IsInt() && arg.ValueInt() >= 0; + } else if constexpr (std::is_same_v) { + return arg.IsDouble(); + } else if constexpr (std::is_same_v) { + return arg.IsNumeric(); + } else if constexpr (std::is_same_v) { + return arg.IsList(); + } else if constexpr (std::is_same_v) { + return arg.IsString(); + } else if constexpr (std::is_same_v) { + return arg.IsMap(); + } else if constexpr (std::is_same_v) { + return arg.IsVertex(); + } else if constexpr (std::is_same_v) { + return arg.IsEdge(); + } else if constexpr (std::is_same_v) { + return arg.IsPath(); + } else if constexpr (std::is_same_v) { + return arg.IsDate(); + } else if constexpr (std::is_same_v) { + return arg.IsLocalTime(); + } else if constexpr (std::is_same_v) { + return arg.IsLocalDateTime(); + } else if constexpr (std::is_same_v) { + return arg.IsDuration(); + } else if constexpr (std::is_same_v) { + return true; + } else { + static_assert(std::is_same_v, "Unknown ArgType"); + } + return false; +} + +template +constexpr const char *ArgTypeName() { + // The type names returned should be standardized openCypher type names. + // https://github.com/opencypher/openCypher/blob/master/docs/openCypher9.pdf + if constexpr (std::is_same_v) { + return "null"; + } else if constexpr (std::is_same_v) { + return "boolean"; + } else if constexpr (std::is_same_v) { + return "integer"; + } else if constexpr (std::is_same_v) { + return "positive integer"; + } else if constexpr (std::is_same_v) { + return "non-zero integer"; + } else if constexpr (std::is_same_v) { + return "non-negative integer"; + } else if constexpr (std::is_same_v) { + return "float"; + } else if constexpr (std::is_same_v) { + return "number"; + } else if constexpr (std::is_same_v) { + return "list"; + } else if constexpr (std::is_same_v) { + return "string"; + } else if constexpr (std::is_same_v) { + return "map"; + } else if constexpr (std::is_same_v) { + return "node"; + } else if constexpr (std::is_same_v) { + return "relationship"; + } else if constexpr (std::is_same_v) { + return "path"; + } else if constexpr (std::is_same_v) { + return "void"; + } else if constexpr (std::is_same_v) { + return "Date"; + } else if constexpr (std::is_same_v) { + return "LocalTime"; + } else if constexpr (std::is_same_v) { + return "LocalDateTime"; + } else if constexpr (std::is_same_v) { + return "Duration"; + } else { + static_assert(std::is_same_v, "Unknown ArgType"); + } + return ""; +} + +template +struct Or; + +template +struct Or { + static bool Check(const TypedValue &arg) { return ArgIsType(arg); } + + static std::string TypeNames() { return ArgTypeName(); } +}; + +template +struct Or { + static bool Check(const TypedValue &arg) { + if (ArgIsType(arg)) return true; + return Or::Check(arg); + } + + static std::string TypeNames() { + if constexpr (sizeof...(ArgTypes) > 1) { + return fmt::format("'{}', {}", ArgTypeName(), Or::TypeNames()); + } else { + return fmt::format("'{}' or '{}'", ArgTypeName(), Or::TypeNames()); + } + } +}; + +template +struct IsOrType { + static constexpr bool value = false; +}; + +template +struct IsOrType> { + static constexpr bool value = true; +}; + +template +struct Optional; + +template +struct Optional { + static constexpr size_t size = 1; + + static void Check(const char *name, const TypedValue *args, int64_t nargs, int64_t pos) { + if (nargs == 0) return; + const TypedValue &arg = args[0]; + if constexpr (IsOrType::value) { + if (!ArgType::Check(arg)) { + throw QueryRuntimeException("Optional '{}' argument at position {} must be either {}.", name, pos, + ArgType::TypeNames()); + } + } else { + if (!ArgIsType(arg)) + throw QueryRuntimeException("Optional '{}' argument at position {} must be '{}'.", name, pos, + ArgTypeName()); + } + } +}; + +template +struct Optional { + static constexpr size_t size = 1 + sizeof...(ArgTypes); + + static void Check(const char *name, const TypedValue *args, int64_t nargs, int64_t pos) { + if (nargs == 0) return; + Optional::Check(name, args, nargs, pos); + Optional::Check(name, args + 1, nargs - 1, pos + 1); + } +}; + +template +struct IsOptional { + static constexpr bool value = false; +}; + +template +struct IsOptional> { + static constexpr bool value = true; +}; + +template +constexpr size_t FTypeRequiredArgs() { + if constexpr (IsOptional::value) { + static_assert(sizeof...(ArgTypes) == 0, "Optional arguments must be last!"); + return 0; + } else if constexpr (sizeof...(ArgTypes) == 0) { + return 1; + } else { + return 1U + FTypeRequiredArgs(); + } +} + +template +constexpr size_t FTypeOptionalArgs() { + if constexpr (IsOptional::value) { + static_assert(sizeof...(ArgTypes) == 0, "Optional arguments must be last!"); + return ArgType::size; + } else if constexpr (sizeof...(ArgTypes) == 0) { + return 0; + } else { + return FTypeOptionalArgs(); + } +} + +template +void FType(const char *name, const TypedValue *args, int64_t nargs, int64_t pos = 1) { + if constexpr (std::is_same_v) { + if (nargs != 0) { + throw QueryRuntimeException("'{}' requires no arguments.", name); + } + return; + } + static constexpr int64_t required_args = FTypeRequiredArgs(); + static constexpr int64_t optional_args = FTypeOptionalArgs(); + static constexpr int64_t total_args = required_args + optional_args; + if constexpr (optional_args > 0) { + if (nargs < required_args || nargs > total_args) { + throw QueryRuntimeException("'{}' requires between {} and {} arguments.", name, required_args, total_args); + } + } else { + if (nargs != required_args) { + throw QueryRuntimeException("'{}' requires exactly {} {}.", name, required_args, + required_args == 1 ? "argument" : "arguments"); + } + } + const TypedValue &arg = args[0]; + if constexpr (IsOrType::value) { + if (!ArgType::Check(arg)) { + throw QueryRuntimeException("'{}' argument at position {} must be either {}.", name, pos, ArgType::TypeNames()); + } + } else if constexpr (IsOptional::value) { + static_assert(sizeof...(ArgTypes) == 0, "Optional arguments must be last!"); + ArgType::Check(name, args, nargs, pos); + } else { + if (!ArgIsType(arg)) { + throw QueryRuntimeException("'{}' argument at position {} must be '{}'", name, pos, ArgTypeName()); + } + } + if constexpr (sizeof...(ArgTypes) > 0) { + FType(name, args + 1, nargs - 1, pos + 1); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// END function type description eDSL +//////////////////////////////////////////////////////////////////////////////// + +// Predicate functions. +// Neo4j has all, any, exists, none, single +// Those functions are a little bit different since they take a filterExpression +// as an argument. +// There is all, any, none and single productions in opencypher grammar, but it +// will be trivial to also add exists. +// TODO: Implement this. + +// Scalar functions. +// We don't have a way to implement id function since we don't store any. If it +// is really neccessary we could probably map vlist* to id. +// TODO: Implement length (it works on a path, but we didn't define path +// structure yet). +// TODO: Implement size(pattern), for example size((a)-[:X]-()) should return +// number of results of this pattern. I don't think we will ever do this. +// TODO: Implement rest of the list functions. +// TODO: Implement degrees, haversin, radians +// TODO: Implement spatial functions + +TypedValue EndNode(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("endNode", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + return TypedValue(args[0].ValueEdge().To(), ctx.memory); +} + +TypedValue Head(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("head", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + const auto &list = args[0].ValueList(); + if (list.empty()) return TypedValue(ctx.memory); + return TypedValue(list[0], ctx.memory); +} + +TypedValue Last(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("last", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + const auto &list = args[0].ValueList(); + if (list.empty()) return TypedValue(ctx.memory); + return TypedValue(list.back(), ctx.memory); +} + +TypedValue Properties(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("properties", args, nargs); + auto *dba = ctx.db_accessor; + auto get_properties = [&](const auto &record_accessor) { + TypedValue::TMap properties(ctx.memory); + auto maybe_props = record_accessor.Properties(ctx.view); + if (maybe_props.HasError()) { + switch (maybe_props.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to get properties from a deleted object."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get properties from an object that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when getting properties."); + } + } + for (const auto &property : *maybe_props) { + properties.emplace(dba->PropertyToName(property.first), property.second); + } + return TypedValue(std::move(properties)); + }; + const auto &value = args[0]; + if (value.IsNull()) { + return TypedValue(ctx.memory); + } else if (value.IsVertex()) { + return get_properties(value.ValueVertex()); + } else { + return get_properties(value.ValueEdge()); + } +} + +TypedValue Size(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("size", args, nargs); + const auto &value = args[0]; + if (value.IsNull()) { + return TypedValue(ctx.memory); + } else if (value.IsList()) { + return TypedValue(static_cast(value.ValueList().size()), ctx.memory); + } else if (value.IsString()) { + return TypedValue(static_cast(value.ValueString().size()), ctx.memory); + } else if (value.IsMap()) { + // neo4j doesn't implement size for map, but I don't see a good reason not + // to do it. + return TypedValue(static_cast(value.ValueMap().size()), ctx.memory); + } else { + return TypedValue(static_cast(value.ValuePath().edges().size()), ctx.memory); + } +} + +TypedValue StartNode(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("startNode", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + return TypedValue(args[0].ValueEdge().From(), ctx.memory); +} + +namespace { + +size_t UnwrapDegreeResult(storage::v3::Result maybe_degree) { + if (maybe_degree.HasError()) { + switch (maybe_degree.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to get degree of a deleted node."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get degree of a node that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when getting node degree."); + } + } + return *maybe_degree; +} + +} // namespace + +TypedValue Degree(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("degree", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + const auto &vertex = args[0].ValueVertex(); + size_t out_degree = UnwrapDegreeResult(vertex.OutDegree(ctx.view)); + size_t in_degree = UnwrapDegreeResult(vertex.InDegree(ctx.view)); + return TypedValue(static_cast(out_degree + in_degree), ctx.memory); +} + +TypedValue InDegree(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("inDegree", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + const auto &vertex = args[0].ValueVertex(); + size_t in_degree = UnwrapDegreeResult(vertex.InDegree(ctx.view)); + return TypedValue(static_cast(in_degree), ctx.memory); +} + +TypedValue OutDegree(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("outDegree", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + const auto &vertex = args[0].ValueVertex(); + size_t out_degree = UnwrapDegreeResult(vertex.OutDegree(ctx.view)); + return TypedValue(static_cast(out_degree), ctx.memory); +} + +TypedValue ToBoolean(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("toBoolean", args, nargs); + const auto &value = args[0]; + if (value.IsNull()) { + return TypedValue(ctx.memory); + } else if (value.IsBool()) { + return TypedValue(value.ValueBool(), ctx.memory); + } else if (value.IsInt()) { + return TypedValue(value.ValueInt() != 0L, ctx.memory); + } else { + auto s = utils::ToUpperCase(utils::Trim(value.ValueString())); + if (s == "TRUE") return TypedValue(true, ctx.memory); + if (s == "FALSE") return TypedValue(false, ctx.memory); + // I think this is just stupid and that exception should be thrown, but + // neo4j does it this way... + return TypedValue(ctx.memory); + } +} + +TypedValue ToFloat(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("toFloat", args, nargs); + const auto &value = args[0]; + if (value.IsNull()) { + return TypedValue(ctx.memory); + } else if (value.IsInt()) { + return TypedValue(static_cast(value.ValueInt()), ctx.memory); + } else if (value.IsDouble()) { + return TypedValue(value, ctx.memory); + } else { + try { + return TypedValue(utils::ParseDouble(utils::Trim(value.ValueString())), ctx.memory); + } catch (const utils::BasicException &) { + return TypedValue(ctx.memory); + } + } +} + +TypedValue ToInteger(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("toInteger", args, nargs); + const auto &value = args[0]; + if (value.IsNull()) { + return TypedValue(ctx.memory); + } else if (value.IsBool()) { + return TypedValue(value.ValueBool() ? 1L : 0L, ctx.memory); + } else if (value.IsInt()) { + return TypedValue(value, ctx.memory); + } else if (value.IsDouble()) { + return TypedValue(static_cast(value.ValueDouble()), ctx.memory); + } else { + try { + // Yup, this is correct. String is valid if it has floating point + // number, then it is parsed and converted to int. + return TypedValue(static_cast(utils::ParseDouble(utils::Trim(value.ValueString()))), ctx.memory); + } catch (const utils::BasicException &) { + return TypedValue(ctx.memory); + } + } +} + +TypedValue Type(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("type", args, nargs); + auto *dba = ctx.db_accessor; + if (args[0].IsNull()) return TypedValue(ctx.memory); + return TypedValue(dba->EdgeTypeToName(args[0].ValueEdge().EdgeType()), ctx.memory); +} + +TypedValue ValueType(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("type", args, nargs); + // The type names returned should be standardized openCypher type names. + // https://github.com/opencypher/openCypher/blob/master/docs/openCypher9.pdf + switch (args[0].type()) { + case TypedValue::Type::Null: + return TypedValue("NULL", ctx.memory); + case TypedValue::Type::Bool: + return TypedValue("BOOLEAN", ctx.memory); + case TypedValue::Type::Int: + return TypedValue("INTEGER", ctx.memory); + case TypedValue::Type::Double: + return TypedValue("FLOAT", ctx.memory); + case TypedValue::Type::String: + return TypedValue("STRING", ctx.memory); + case TypedValue::Type::List: + return TypedValue("LIST", ctx.memory); + case TypedValue::Type::Map: + return TypedValue("MAP", ctx.memory); + case TypedValue::Type::Vertex: + return TypedValue("NODE", ctx.memory); + case TypedValue::Type::Edge: + return TypedValue("RELATIONSHIP", ctx.memory); + case TypedValue::Type::Path: + return TypedValue("PATH", ctx.memory); + case TypedValue::Type::Date: + return TypedValue("DATE", ctx.memory); + case TypedValue::Type::LocalTime: + return TypedValue("LOCAL_TIME", ctx.memory); + case TypedValue::Type::LocalDateTime: + return TypedValue("LOCAL_DATE_TIME", ctx.memory); + case TypedValue::Type::Duration: + return TypedValue("DURATION", ctx.memory); + } +} + +// TODO: How is Keys different from Properties function? +TypedValue Keys(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("keys", args, nargs); + auto *dba = ctx.db_accessor; + auto get_keys = [&](const auto &record_accessor) { + TypedValue::TVector keys(ctx.memory); + auto maybe_props = record_accessor.Properties(ctx.view); + if (maybe_props.HasError()) { + switch (maybe_props.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to get keys from a deleted object."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get keys from an object that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when getting keys."); + } + } + for (const auto &property : *maybe_props) { + keys.emplace_back(dba->PropertyToName(property.first)); + } + return TypedValue(std::move(keys)); + }; + const auto &value = args[0]; + if (value.IsNull()) { + return TypedValue(ctx.memory); + } else if (value.IsVertex()) { + return get_keys(value.ValueVertex()); + } else { + return get_keys(value.ValueEdge()); + } +} + +TypedValue Labels(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("labels", args, nargs); + auto *dba = ctx.db_accessor; + if (args[0].IsNull()) return TypedValue(ctx.memory); + TypedValue::TVector labels(ctx.memory); + auto maybe_labels = args[0].ValueVertex().Labels(ctx.view); + if (maybe_labels.HasError()) { + switch (maybe_labels.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to get labels from a deleted node."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get labels from a node that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when getting labels."); + } + } + for (const auto &label : *maybe_labels) { + labels.emplace_back(dba->LabelToName(label)); + } + return TypedValue(std::move(labels)); +} + +TypedValue Nodes(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("nodes", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + const auto &vertices = args[0].ValuePath().vertices(); + TypedValue::TVector values(ctx.memory); + values.reserve(vertices.size()); + for (const auto &v : vertices) values.emplace_back(v); + return TypedValue(std::move(values)); +} + +TypedValue Relationships(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("relationships", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + const auto &edges = args[0].ValuePath().edges(); + TypedValue::TVector values(ctx.memory); + values.reserve(edges.size()); + for (const auto &e : edges) values.emplace_back(e); + return TypedValue(std::move(values)); +} + +TypedValue Range(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, Or, Optional>>("range", args, nargs); + for (int64_t i = 0; i < nargs; ++i) + if (args[i].IsNull()) return TypedValue(ctx.memory); + auto lbound = args[0].ValueInt(); + auto rbound = args[1].ValueInt(); + int64_t step = nargs == 3 ? args[2].ValueInt() : 1; + TypedValue::TVector list(ctx.memory); + if (lbound <= rbound && step > 0) { + for (auto i = lbound; i <= rbound; i += step) { + list.emplace_back(i); + } + } else if (lbound >= rbound && step < 0) { + for (auto i = lbound; i >= rbound; i += step) { + list.emplace_back(i); + } + } + return TypedValue(std::move(list)); +} + +TypedValue Tail(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("tail", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + TypedValue::TVector list(args[0].ValueList(), ctx.memory); + if (list.empty()) return TypedValue(std::move(list)); + list.erase(list.begin()); + return TypedValue(std::move(list)); +} + +TypedValue UniformSample(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, Or>("uniformSample", args, nargs); + static thread_local std::mt19937 pseudo_rand_gen_{std::random_device{}()}; + if (args[0].IsNull() || args[1].IsNull()) return TypedValue(ctx.memory); + const auto &population = args[0].ValueList(); + auto population_size = population.size(); + if (population_size == 0) return TypedValue(ctx.memory); + auto desired_length = args[1].ValueInt(); + std::uniform_int_distribution rand_dist{0, population_size - 1}; + TypedValue::TVector sampled(ctx.memory); + sampled.reserve(desired_length); + for (int64_t i = 0; i < desired_length; ++i) { + sampled.emplace_back(population[rand_dist(pseudo_rand_gen_)]); + } + return TypedValue(std::move(sampled)); +} + +TypedValue Abs(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("abs", args, nargs); + const auto &value = args[0]; + if (value.IsNull()) { + return TypedValue(ctx.memory); + } else if (value.IsInt()) { + return TypedValue(std::abs(value.ValueInt()), ctx.memory); + } else { + return TypedValue(std::abs(value.ValueDouble()), ctx.memory); + } +} + +#define WRAP_CMATH_FLOAT_FUNCTION(name, lowercased_name) \ + TypedValue name(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { \ + FType>(#lowercased_name, args, nargs); \ + const auto &value = args[0]; \ + if (value.IsNull()) { \ + return TypedValue(ctx.memory); \ + } else if (value.IsInt()) { \ + return TypedValue(lowercased_name(value.ValueInt()), ctx.memory); \ + } else { \ + return TypedValue(lowercased_name(value.ValueDouble()), ctx.memory); \ + } \ + } + +WRAP_CMATH_FLOAT_FUNCTION(Ceil, ceil) +WRAP_CMATH_FLOAT_FUNCTION(Floor, floor) +// We are not completely compatible with neoj4 in this function because, +// neo4j rounds -0.5, -1.5, -2.5... to 0, -1, -2... +WRAP_CMATH_FLOAT_FUNCTION(Round, round) +WRAP_CMATH_FLOAT_FUNCTION(Exp, exp) +WRAP_CMATH_FLOAT_FUNCTION(Log, log) +WRAP_CMATH_FLOAT_FUNCTION(Log10, log10) +WRAP_CMATH_FLOAT_FUNCTION(Sqrt, sqrt) +WRAP_CMATH_FLOAT_FUNCTION(Acos, acos) +WRAP_CMATH_FLOAT_FUNCTION(Asin, asin) +WRAP_CMATH_FLOAT_FUNCTION(Atan, atan) +WRAP_CMATH_FLOAT_FUNCTION(Cos, cos) +WRAP_CMATH_FLOAT_FUNCTION(Sin, sin) +WRAP_CMATH_FLOAT_FUNCTION(Tan, tan) + +#undef WRAP_CMATH_FLOAT_FUNCTION + +TypedValue Atan2(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, Or>("atan2", args, nargs); + if (args[0].IsNull() || args[1].IsNull()) return TypedValue(ctx.memory); + auto to_double = [](const TypedValue &t) -> double { + if (t.IsInt()) { + return t.ValueInt(); + } else { + return t.ValueDouble(); + } + }; + double y = to_double(args[0]); + double x = to_double(args[1]); + return TypedValue(atan2(y, x), ctx.memory); +} + +TypedValue Sign(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("sign", args, nargs); + auto sign = [&](auto x) { return TypedValue((0 < x) - (x < 0), ctx.memory); }; + const auto &value = args[0]; + if (value.IsNull()) { + return TypedValue(ctx.memory); + } else if (value.IsInt()) { + return sign(value.ValueInt()); + } else { + return sign(value.ValueDouble()); + } +} + +TypedValue E(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType("e", args, nargs); + return TypedValue(M_E, ctx.memory); +} + +TypedValue Pi(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType("pi", args, nargs); + return TypedValue(M_PI, ctx.memory); +} + +TypedValue Rand(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType("rand", args, nargs); + static thread_local std::mt19937 pseudo_rand_gen_{std::random_device{}()}; + static thread_local std::uniform_real_distribution<> rand_dist_{0, 1}; + return TypedValue(rand_dist_(pseudo_rand_gen_), ctx.memory); +} + +template +TypedValue StringMatchOperator(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, Or>(TPredicate::name, args, nargs); + if (args[0].IsNull() || args[1].IsNull()) return TypedValue(ctx.memory); + const auto &s1 = args[0].ValueString(); + const auto &s2 = args[1].ValueString(); + return TypedValue(TPredicate{}(s1, s2), ctx.memory); +} + +// Check if s1 starts with s2. +struct StartsWithPredicate { + static constexpr const char *name = "startsWith"; + bool operator()(const TypedValue::TString &s1, const TypedValue::TString &s2) const { + if (s1.size() < s2.size()) return false; + return std::equal(s2.begin(), s2.end(), s1.begin()); + } +}; +auto StartsWith = StringMatchOperator; + +// Check if s1 ends with s2. +struct EndsWithPredicate { + static constexpr const char *name = "endsWith"; + bool operator()(const TypedValue::TString &s1, const TypedValue::TString &s2) const { + if (s1.size() < s2.size()) return false; + return std::equal(s2.rbegin(), s2.rend(), s1.rbegin()); + } +}; +auto EndsWith = StringMatchOperator; + +// Check if s1 contains s2. +struct ContainsPredicate { + static constexpr const char *name = "contains"; + bool operator()(const TypedValue::TString &s1, const TypedValue::TString &s2) const { + if (s1.size() < s2.size()) return false; + return s1.find(s2) != std::string::npos; + } +}; +auto Contains = StringMatchOperator; + +TypedValue Assert(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("assert", args, nargs); + if (!args[0].ValueBool()) { + std::string message("Assertion failed"); + if (nargs == 2) { + message += ": "; + message += args[1].ValueString(); + } + message += "."; + throw QueryRuntimeException(message); + } + return TypedValue(args[0], ctx.memory); +} + +TypedValue Counter(const TypedValue *args, int64_t nargs, const FunctionContext &context) { + FType>("counter", args, nargs); + int64_t step = 1; + if (nargs == 3) { + step = args[2].ValueInt(); + } + + auto [it, inserted] = context.counters->emplace(args[0].ValueString(), args[1].ValueInt()); + auto value = it->second; + it->second += step; + + return TypedValue(value, context.memory); +} + +TypedValue Id(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("id", args, nargs); + const auto &arg = args[0]; + if (arg.IsNull()) { + return TypedValue(ctx.memory); + } else if (arg.IsVertex()) { + return TypedValue(arg.ValueVertex().CypherId(), ctx.memory); + } else { + return TypedValue(arg.ValueEdge().CypherId(), ctx.memory); + } +} + +TypedValue ToString(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("toString", args, nargs); + const auto &arg = args[0]; + if (arg.IsNull()) { + return TypedValue(ctx.memory); + } else if (arg.IsString()) { + return TypedValue(arg, ctx.memory); + } else if (arg.IsInt()) { + // TODO: This is making a pointless copy of std::string, we may want to + // use a different conversion to string + return TypedValue(std::to_string(arg.ValueInt()), ctx.memory); + } else if (arg.IsDouble()) { + return TypedValue(std::to_string(arg.ValueDouble()), ctx.memory); + } else { + return TypedValue(arg.ValueBool() ? "true" : "false", ctx.memory); + } +} + +TypedValue Timestamp(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>>("timestamp", args, nargs); + const auto &arg = *args; + if (arg.IsDate()) { + return TypedValue(arg.ValueDate().MicrosecondsSinceEpoch(), ctx.memory); + } + if (arg.IsLocalTime()) { + return TypedValue(arg.ValueLocalTime().MicrosecondsSinceEpoch(), ctx.memory); + } + if (arg.IsLocalDateTime()) { + return TypedValue(arg.ValueLocalDateTime().MicrosecondsSinceEpoch(), ctx.memory); + } + if (arg.IsDuration()) { + return TypedValue(arg.ValueDuration().microseconds, ctx.memory); + } + return TypedValue(ctx.timestamp, ctx.memory); +} + +TypedValue Left(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, Or>("left", args, nargs); + if (args[0].IsNull() || args[1].IsNull()) return TypedValue(ctx.memory); + return TypedValue(utils::Substr(args[0].ValueString(), 0, args[1].ValueInt()), ctx.memory); +} + +TypedValue Right(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, Or>("right", args, nargs); + if (args[0].IsNull() || args[1].IsNull()) return TypedValue(ctx.memory); + const auto &str = args[0].ValueString(); + auto len = args[1].ValueInt(); + return len <= str.size() ? TypedValue(utils::Substr(str, str.size() - len, len), ctx.memory) + : TypedValue(str, ctx.memory); +} + +TypedValue CallStringFunction(const TypedValue *args, int64_t nargs, utils::MemoryResource *memory, const char *name, + std::function fun) { + FType>(name, args, nargs); + if (args[0].IsNull()) return TypedValue(memory); + return TypedValue(fun(args[0].ValueString()), memory); +} + +TypedValue LTrim(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + return CallStringFunction(args, nargs, ctx.memory, "lTrim", + [&](const auto &str) { return TypedValue::TString(utils::LTrim(str), ctx.memory); }); +} + +TypedValue RTrim(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + return CallStringFunction(args, nargs, ctx.memory, "rTrim", + [&](const auto &str) { return TypedValue::TString(utils::RTrim(str), ctx.memory); }); +} + +TypedValue Trim(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + return CallStringFunction(args, nargs, ctx.memory, "trim", + [&](const auto &str) { return TypedValue::TString(utils::Trim(str), ctx.memory); }); +} + +TypedValue Reverse(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + return CallStringFunction(args, nargs, ctx.memory, "reverse", + [&](const auto &str) { return utils::Reversed(str, ctx.memory); }); +} + +TypedValue ToLower(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + return CallStringFunction(args, nargs, ctx.memory, "toLower", [&](const auto &str) { + TypedValue::TString res(ctx.memory); + utils::ToLowerCase(&res, str); + return res; + }); +} + +TypedValue ToUpper(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + return CallStringFunction(args, nargs, ctx.memory, "toUpper", [&](const auto &str) { + TypedValue::TString res(ctx.memory); + utils::ToUpperCase(&res, str); + return res; + }); +} + +TypedValue Replace(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, Or, Or>("replace", args, nargs); + if (args[0].IsNull() || args[1].IsNull() || args[2].IsNull()) { + return TypedValue(ctx.memory); + } + TypedValue::TString replaced(ctx.memory); + utils::Replace(&replaced, args[0].ValueString(), args[1].ValueString(), args[2].ValueString()); + return TypedValue(std::move(replaced)); +} + +TypedValue Split(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, Or>("split", args, nargs); + if (args[0].IsNull() || args[1].IsNull()) { + return TypedValue(ctx.memory); + } + TypedValue::TVector result(ctx.memory); + utils::Split(&result, args[0].ValueString(), args[1].ValueString()); + return TypedValue(std::move(result)); +} + +TypedValue Substring(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType, NonNegativeInteger, Optional>("substring", args, nargs); + if (args[0].IsNull()) return TypedValue(ctx.memory); + const auto &str = args[0].ValueString(); + auto start = args[1].ValueInt(); + if (nargs == 2) return TypedValue(utils::Substr(str, start), ctx.memory); + auto len = args[2].ValueInt(); + return TypedValue(utils::Substr(str, start, len), ctx.memory); +} + +TypedValue ToByteString(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType("toByteString", args, nargs); + const auto &str = args[0].ValueString(); + if (str.empty()) return TypedValue("", ctx.memory); + if (!utils::StartsWith(str, "0x") && !utils::StartsWith(str, "0X")) { + throw QueryRuntimeException("'toByteString' argument must start with '0x'"); + } + const auto &hex_str = utils::Substr(str, 2); + auto read_hex = [](const char ch) -> unsigned char { + if (ch >= '0' && ch <= '9') return ch - '0'; + if (ch >= 'a' && ch <= 'f') return ch - 'a' + 10; + if (ch >= 'A' && ch <= 'F') return ch - 'A' + 10; + throw QueryRuntimeException("'toByteString' argument has an invalid character '{}'", ch); + }; + utils::pmr::string bytes(ctx.memory); + bytes.reserve((1 + hex_str.size()) / 2); + size_t i = 0; + // Treat odd length hex string as having a leading zero. + if (hex_str.size() % 2) bytes.append(1, read_hex(hex_str[i++])); + for (; i < hex_str.size(); i += 2) { + unsigned char byte = read_hex(hex_str[i]) * 16U + read_hex(hex_str[i + 1]); + // MemcpyCast in case we are converting to a signed value, so as to avoid + // undefined behaviour. + bytes.append(1, utils::MemcpyCast(byte)); + } + return TypedValue(std::move(bytes)); +} + +TypedValue FromByteString(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("fromByteString", args, nargs); + const auto &bytes = args[0].ValueString(); + if (bytes.empty()) return TypedValue("", ctx.memory); + size_t min_length = bytes.size(); + if (nargs == 2) min_length = std::max(min_length, static_cast(args[1].ValueInt())); + utils::pmr::string str(ctx.memory); + str.reserve(min_length * 2 + 2); + str.append("0x"); + for (size_t pad = 0; pad < min_length - bytes.size(); ++pad) str.append(2, '0'); + // Convert the bytes to a character string in hex representation. + // Unfortunately, we don't know whether the default `char` is signed or + // unsigned, so we have to work around any potential undefined behaviour when + // conversions between the 2 occur. That's why this function is more + // complicated than it should be. + auto to_hex = [](const unsigned char val) -> char { + unsigned char ch = val < 10U ? static_cast('0') + val : static_cast('a') + val - 10U; + return utils::MemcpyCast(ch); + }; + for (unsigned char byte : bytes) { + str.append(1, to_hex(byte / 16U)); + str.append(1, to_hex(byte % 16U)); + } + return TypedValue(std::move(str)); +} + +template +concept IsNumberOrInteger = utils::SameAsAnyOf; + +template +void MapNumericParameters(auto ¶meter_mappings, const auto &input_parameters) { + for (const auto &[key, value] : input_parameters) { + if (auto it = parameter_mappings.find(key); it != parameter_mappings.end()) { + if (value.IsInt()) { + *it->second = value.ValueInt(); + } else if (std::is_same_v && value.IsDouble()) { + *it->second = value.ValueDouble(); + } else { + std::string_view error = std::is_same_v ? "an integer." : "a numeric value."; + throw QueryRuntimeException("Invalid value for key '{}'. Expected {}", key, error); + } + } else { + throw QueryRuntimeException("Unknown key '{}'.", key); + } + } +} + +TypedValue Date(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>>("date", args, nargs); + if (nargs == 0) { + return TypedValue(utils::LocalDateTime(ctx.timestamp).date, ctx.memory); + } + + if (args[0].IsString()) { + const auto &[date_parameters, is_extended] = utils::ParseDateParameters(args[0].ValueString()); + return TypedValue(utils::Date(date_parameters), ctx.memory); + } + + utils::DateParameters date_parameters; + + using namespace std::literals; + std::unordered_map parameter_mappings = {std::pair{"year"sv, &date_parameters.year}, + std::pair{"month"sv, &date_parameters.month}, + std::pair{"day"sv, &date_parameters.day}}; + + MapNumericParameters(parameter_mappings, args[0].ValueMap()); + return TypedValue(utils::Date(date_parameters), ctx.memory); +} + +TypedValue LocalTime(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>>("localtime", args, nargs); + + if (nargs == 0) { + return TypedValue(utils::LocalDateTime(ctx.timestamp).local_time, ctx.memory); + } + + if (args[0].IsString()) { + const auto &[local_time_parameters, is_extended] = utils::ParseLocalTimeParameters(args[0].ValueString()); + return TypedValue(utils::LocalTime(local_time_parameters), ctx.memory); + } + + utils::LocalTimeParameters local_time_parameters; + + using namespace std::literals; + std::unordered_map parameter_mappings{ + std::pair{"hour"sv, &local_time_parameters.hour}, + std::pair{"minute"sv, &local_time_parameters.minute}, + std::pair{"second"sv, &local_time_parameters.second}, + std::pair{"millisecond"sv, &local_time_parameters.millisecond}, + std::pair{"microsecond"sv, &local_time_parameters.microsecond}, + }; + + MapNumericParameters(parameter_mappings, args[0].ValueMap()); + return TypedValue(utils::LocalTime(local_time_parameters), ctx.memory); +} + +TypedValue LocalDateTime(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>>("localdatetime", args, nargs); + + if (nargs == 0) { + return TypedValue(utils::LocalDateTime(ctx.timestamp), ctx.memory); + } + + if (args[0].IsString()) { + const auto &[date_parameters, local_time_parameters] = ParseLocalDateTimeParameters(args[0].ValueString()); + return TypedValue(utils::LocalDateTime(date_parameters, local_time_parameters), ctx.memory); + } + + utils::DateParameters date_parameters; + utils::LocalTimeParameters local_time_parameters; + using namespace std::literals; + std::unordered_map parameter_mappings{ + std::pair{"year"sv, &date_parameters.year}, + std::pair{"month"sv, &date_parameters.month}, + std::pair{"day"sv, &date_parameters.day}, + std::pair{"hour"sv, &local_time_parameters.hour}, + std::pair{"minute"sv, &local_time_parameters.minute}, + std::pair{"second"sv, &local_time_parameters.second}, + std::pair{"millisecond"sv, &local_time_parameters.millisecond}, + std::pair{"microsecond"sv, &local_time_parameters.microsecond}, + }; + + MapNumericParameters(parameter_mappings, args[0].ValueMap()); + return TypedValue(utils::LocalDateTime(date_parameters, local_time_parameters), ctx.memory); +} + +TypedValue Duration(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) { + FType>("duration", args, nargs); + + if (args[0].IsString()) { + return TypedValue(utils::Duration(ParseDurationParameters(args[0].ValueString())), ctx.memory); + } + + utils::DurationParameters duration_parameters; + using namespace std::literals; + std::unordered_map parameter_mappings{std::pair{"day"sv, &duration_parameters.day}, + std::pair{"hour"sv, &duration_parameters.hour}, + std::pair{"minute"sv, &duration_parameters.minute}, + std::pair{"second"sv, &duration_parameters.second}, + std::pair{"millisecond"sv, &duration_parameters.millisecond}, + std::pair{"microsecond"sv, &duration_parameters.microsecond}}; + MapNumericParameters(parameter_mappings, args[0].ValueMap()); + return TypedValue(utils::Duration(duration_parameters), ctx.memory); +} + +std::function UserFunction( + const mgp_func &func, const std::string &fully_qualified_name) { + return [func, fully_qualified_name](const TypedValue *args, int64_t nargs, const FunctionContext &ctx) -> TypedValue { + /// Find function is called to aquire the lock on Module pointer while user-defined function is executed + const auto &maybe_found = + procedure::FindFunction(procedure::gModuleRegistry, fully_qualified_name, utils::NewDeleteResource()); + if (!maybe_found) { + throw QueryRuntimeException( + "Function '{}' has been unloaded. Please check query modules to confirm that function is loaded in Memgraph.", + fully_qualified_name); + } + /// Explicit extraction of module pointer, to clearly state that the lock is aquired. + // NOLINTNEXTLINE(clang-diagnostic-unused-variable) + const auto &module_ptr = (*maybe_found).first; + + const auto &func_cb = func.cb; + mgp_memory memory{ctx.memory}; + mgp_func_context functx{ctx.db_accessor, ctx.view}; + auto graph = mgp_graph::NonWritableGraph(*ctx.db_accessor, ctx.view); + + std::vector args_list; + args_list.reserve(nargs); + for (std::size_t i = 0; i < nargs; ++i) { + args_list.emplace_back(args[i]); + } + + auto function_argument_list = mgp_list(ctx.memory); + procedure::ConstructArguments(args_list, func, fully_qualified_name, function_argument_list, graph); + + mgp_func_result maybe_res; + func_cb(&function_argument_list, &functx, &maybe_res, &memory); + if (maybe_res.error_msg) { + throw QueryRuntimeException(*maybe_res.error_msg); + } + + if (!maybe_res.value) { + throw QueryRuntimeException( + "Function '{}' didn't set the result nor the error message. Please either set the result by using " + "mgp_func_result_set_value or the error by using mgp_func_result_set_error_msg.", + fully_qualified_name); + } + + return {*(maybe_res.value), ctx.memory}; + }; +} + +} // namespace + +std::function NameToFunction( + const std::string &function_name) { + // Scalar functions + if (function_name == "DEGREE") return Degree; + if (function_name == "INDEGREE") return InDegree; + if (function_name == "OUTDEGREE") return OutDegree; + if (function_name == "ENDNODE") return EndNode; + if (function_name == "HEAD") return Head; + if (function_name == kId) return Id; + if (function_name == "LAST") return Last; + if (function_name == "PROPERTIES") return Properties; + if (function_name == "SIZE") return Size; + if (function_name == "STARTNODE") return StartNode; + if (function_name == "TIMESTAMP") return Timestamp; + if (function_name == "TOBOOLEAN") return ToBoolean; + if (function_name == "TOFLOAT") return ToFloat; + if (function_name == "TOINTEGER") return ToInteger; + if (function_name == "TYPE") return Type; + if (function_name == "VALUETYPE") return ValueType; + + // List functions + if (function_name == "KEYS") return Keys; + if (function_name == "LABELS") return Labels; + if (function_name == "NODES") return Nodes; + if (function_name == "RANGE") return Range; + if (function_name == "RELATIONSHIPS") return Relationships; + if (function_name == "TAIL") return Tail; + if (function_name == "UNIFORMSAMPLE") return UniformSample; + + // Mathematical functions - numeric + if (function_name == "ABS") return Abs; + if (function_name == "CEIL") return Ceil; + if (function_name == "FLOOR") return Floor; + if (function_name == "RAND") return Rand; + if (function_name == "ROUND") return Round; + if (function_name == "SIGN") return Sign; + + // Mathematical functions - logarithmic + if (function_name == "E") return E; + if (function_name == "EXP") return Exp; + if (function_name == "LOG") return Log; + if (function_name == "LOG10") return Log10; + if (function_name == "SQRT") return Sqrt; + + // Mathematical functions - trigonometric + if (function_name == "ACOS") return Acos; + if (function_name == "ASIN") return Asin; + if (function_name == "ATAN") return Atan; + if (function_name == "ATAN2") return Atan2; + if (function_name == "COS") return Cos; + if (function_name == "PI") return Pi; + if (function_name == "SIN") return Sin; + if (function_name == "TAN") return Tan; + + // String functions + if (function_name == kContains) return Contains; + if (function_name == kEndsWith) return EndsWith; + if (function_name == "LEFT") return Left; + if (function_name == "LTRIM") return LTrim; + if (function_name == "REPLACE") return Replace; + if (function_name == "REVERSE") return Reverse; + if (function_name == "RIGHT") return Right; + if (function_name == "RTRIM") return RTrim; + if (function_name == "SPLIT") return Split; + if (function_name == kStartsWith) return StartsWith; + if (function_name == "SUBSTRING") return Substring; + if (function_name == "TOLOWER") return ToLower; + if (function_name == "TOSTRING") return ToString; + if (function_name == "TOUPPER") return ToUpper; + if (function_name == "TRIM") return Trim; + + // Memgraph specific functions + if (function_name == "ASSERT") return Assert; + if (function_name == "COUNTER") return Counter; + if (function_name == "TOBYTESTRING") return ToByteString; + if (function_name == "FROMBYTESTRING") return FromByteString; + + // Functions for temporal types + if (function_name == "DATE") return Date; + if (function_name == "LOCALTIME") return LocalTime; + if (function_name == "LOCALDATETIME") return LocalDateTime; + if (function_name == "DURATION") return Duration; + + const auto &maybe_found = + procedure::FindFunction(procedure::gModuleRegistry, function_name, utils::NewDeleteResource()); + + if (maybe_found) { + const auto *func = (*maybe_found).second; + return UserFunction(*func, function_name); + } + + return nullptr; +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/interpret/awesome_memgraph_functions.hpp b/src/query/v2/interpret/awesome_memgraph_functions.hpp new file mode 100644 index 000000000..27c0bc50c --- /dev/null +++ b/src/query/v2/interpret/awesome_memgraph_functions.hpp @@ -0,0 +1,50 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +#include "storage/v3/view.hpp" +#include "utils/memory.hpp" + +namespace memgraph::query::v2 { + +class DbAccessor; +class TypedValue; + +namespace { +const char kStartsWith[] = "STARTSWITH"; +const char kEndsWith[] = "ENDSWITH"; +const char kContains[] = "CONTAINS"; +const char kId[] = "ID"; +} // namespace + +struct FunctionContext { + DbAccessor *db_accessor; + utils::MemoryResource *memory; + int64_t timestamp; + std::unordered_map *counters; + storage::v3::View view; +}; + +/// Return the function implementation with the given name. +/// +/// Note, returned function signature uses C-style access to an array to allow +/// having an array stored anywhere the caller likes, as long as it is +/// contiguous in memory. Since most functions don't take many arguments, it's +/// convenient to have them stored in the calling stack frame. +std::function +NameToFunction(const std::string &function_name); + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/interpret/eval.cpp b/src/query/v2/interpret/eval.cpp new file mode 100644 index 000000000..eba77adf9 --- /dev/null +++ b/src/query/v2/interpret/eval.cpp @@ -0,0 +1,35 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/interpret/eval.hpp" + +namespace memgraph::query::v2 { + +int64_t EvaluateInt(ExpressionEvaluator *evaluator, Expression *expr, const std::string &what) { + TypedValue value = expr->Accept(*evaluator); + try { + return value.ValueInt(); + } catch (TypedValueException &e) { + throw QueryRuntimeException(what + " must be an int"); + } +} + +std::optional EvaluateMemoryLimit(ExpressionEvaluator *eval, Expression *memory_limit, size_t memory_scale) { + if (!memory_limit) return std::nullopt; + auto limit_value = memory_limit->Accept(*eval); + if (!limit_value.IsInt() || limit_value.ValueInt() <= 0) + throw QueryRuntimeException("Memory limit must be a non-negative integer."); + size_t limit = limit_value.ValueInt(); + if (std::numeric_limits::max() / memory_scale < limit) throw QueryRuntimeException("Memory limit overflow."); + return limit * memory_scale; +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/interpret/eval.hpp b/src/query/v2/interpret/eval.hpp new file mode 100644 index 000000000..10cee31a9 --- /dev/null +++ b/src/query/v2/interpret/eval.hpp @@ -0,0 +1,764 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +/// @file +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "query/v2/common.hpp" +#include "query/v2/context.hpp" +#include "query/v2/db_accessor.hpp" +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/semantic/symbol_table.hpp" +#include "query/v2/interpret/frame.hpp" +#include "query/v2/typed_value.hpp" +#include "utils/exceptions.hpp" + +namespace memgraph::query::v2 { + +class ExpressionEvaluator : public ExpressionVisitor { + public: + ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba, + storage::v3::View view) + : frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {} + + using ExpressionVisitor::Visit; + + utils::MemoryResource *GetMemoryResource() const { return ctx_->memory; } + + TypedValue Visit(NamedExpression &named_expression) override { + const auto &symbol = symbol_table_->at(named_expression); + auto value = named_expression.expression_->Accept(*this); + frame_->at(symbol) = value; + return value; + } + + TypedValue Visit(Identifier &ident) override { + return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory); + } + +#define BINARY_OPERATOR_VISITOR(OP_NODE, CPP_OP, CYPHER_OP) \ + TypedValue Visit(OP_NODE &op) override { \ + auto val1 = op.expression1_->Accept(*this); \ + auto val2 = op.expression2_->Accept(*this); \ + try { \ + return val1 CPP_OP val2; \ + } catch (const TypedValueException &) { \ + throw QueryRuntimeException("Invalid types: {} and {} for '{}'.", val1.type(), val2.type(), #CYPHER_OP); \ + } \ + } + +#define UNARY_OPERATOR_VISITOR(OP_NODE, CPP_OP, CYPHER_OP) \ + TypedValue Visit(OP_NODE &op) override { \ + auto val = op.expression_->Accept(*this); \ + try { \ + return CPP_OP val; \ + } catch (const TypedValueException &) { \ + throw QueryRuntimeException("Invalid type {} for '{}'.", val.type(), #CYPHER_OP); \ + } \ + } + + BINARY_OPERATOR_VISITOR(OrOperator, ||, OR); + BINARY_OPERATOR_VISITOR(XorOperator, ^, XOR); + BINARY_OPERATOR_VISITOR(AdditionOperator, +, +); + BINARY_OPERATOR_VISITOR(SubtractionOperator, -, -); + BINARY_OPERATOR_VISITOR(MultiplicationOperator, *, *); + BINARY_OPERATOR_VISITOR(DivisionOperator, /, /); + BINARY_OPERATOR_VISITOR(ModOperator, %, %); + BINARY_OPERATOR_VISITOR(NotEqualOperator, !=, <>); + BINARY_OPERATOR_VISITOR(EqualOperator, ==, =); + BINARY_OPERATOR_VISITOR(LessOperator, <, <); + BINARY_OPERATOR_VISITOR(GreaterOperator, >, >); + BINARY_OPERATOR_VISITOR(LessEqualOperator, <=, <=); + BINARY_OPERATOR_VISITOR(GreaterEqualOperator, >=, >=); + + UNARY_OPERATOR_VISITOR(NotOperator, !, NOT); + UNARY_OPERATOR_VISITOR(UnaryPlusOperator, +, +); + UNARY_OPERATOR_VISITOR(UnaryMinusOperator, -, -); + +#undef BINARY_OPERATOR_VISITOR +#undef UNARY_OPERATOR_VISITOR + + TypedValue Visit(AndOperator &op) override { + auto value1 = op.expression1_->Accept(*this); + if (value1.IsBool() && !value1.ValueBool()) { + // If first expression is false, don't evaluate the second one. + return value1; + } + auto value2 = op.expression2_->Accept(*this); + try { + return value1 && value2; + } catch (const TypedValueException &) { + throw QueryRuntimeException("Invalid types: {} and {} for AND.", value1.type(), value2.type()); + } + } + + TypedValue Visit(IfOperator &if_operator) override { + auto condition = if_operator.condition_->Accept(*this); + if (condition.IsNull()) { + return if_operator.then_expression_->Accept(*this); + } + if (condition.type() != TypedValue::Type::Bool) { + // At the moment IfOperator is used only in CASE construct. + throw QueryRuntimeException("CASE expected boolean expression, got {}.", condition.type()); + } + if (condition.ValueBool()) { + return if_operator.then_expression_->Accept(*this); + } + return if_operator.else_expression_->Accept(*this); + } + + TypedValue Visit(InListOperator &in_list) override { + auto literal = in_list.expression1_->Accept(*this); + auto _list = in_list.expression2_->Accept(*this); + if (_list.IsNull()) { + return TypedValue(ctx_->memory); + } + // Exceptions have higher priority than returning nulls when list expression + // is not null. + if (_list.type() != TypedValue::Type::List) { + throw QueryRuntimeException("IN expected a list, got {}.", _list.type()); + } + const auto &list = _list.ValueList(); + + // If literal is NULL there is no need to try to compare it with every + // element in the list since result of every comparison will be NULL. There + // is one special case that we must test explicitly: if list is empty then + // result is false since no comparison will be performed. + if (list.empty()) return TypedValue(false, ctx_->memory); + if (literal.IsNull()) return TypedValue(ctx_->memory); + + auto has_null = false; + for (const auto &element : list) { + auto result = literal == element; + if (result.IsNull()) { + has_null = true; + } else if (result.ValueBool()) { + return TypedValue(true, ctx_->memory); + } + } + if (has_null) { + return TypedValue(ctx_->memory); + } + return TypedValue(false, ctx_->memory); + } + + TypedValue Visit(SubscriptOperator &list_indexing) override { + auto lhs = list_indexing.expression1_->Accept(*this); + auto index = list_indexing.expression2_->Accept(*this); + if (!lhs.IsList() && !lhs.IsMap() && !lhs.IsVertex() && !lhs.IsEdge() && !lhs.IsNull()) + throw QueryRuntimeException( + "Expected a list, a map, a node or an edge to index with '[]', got " + "{}.", + lhs.type()); + if (lhs.IsNull() || index.IsNull()) return TypedValue(ctx_->memory); + if (lhs.IsList()) { + if (!index.IsInt()) throw QueryRuntimeException("Expected an integer as a list index, got {}.", index.type()); + auto index_int = index.ValueInt(); + // NOTE: Take non-const reference to list, so that we can move out the + // indexed element as the result. + auto &list = lhs.ValueList(); + if (index_int < 0) { + index_int += static_cast(list.size()); + } + if (index_int >= static_cast(list.size()) || index_int < 0) return TypedValue(ctx_->memory); + // NOTE: Explicit move is needed, so that we return the move constructed + // value and preserve the correct MemoryResource. + return std::move(list[index_int]); + } + + if (lhs.IsMap()) { + if (!index.IsString()) throw QueryRuntimeException("Expected a string as a map index, got {}.", index.type()); + // NOTE: Take non-const reference to map, so that we can move out the + // looked-up element as the result. + auto &map = lhs.ValueMap(); + auto found = map.find(index.ValueString()); + if (found == map.end()) return TypedValue(ctx_->memory); + // NOTE: Explicit move is needed, so that we return the move constructed + // value and preserve the correct MemoryResource. + return std::move(found->second); + } + + if (lhs.IsVertex()) { + if (!index.IsString()) throw QueryRuntimeException("Expected a string as a property name, got {}.", index.type()); + return TypedValue(GetProperty(lhs.ValueVertex(), index.ValueString()), ctx_->memory); + } + + if (lhs.IsEdge()) { + if (!index.IsString()) throw QueryRuntimeException("Expected a string as a property name, got {}.", index.type()); + return TypedValue(GetProperty(lhs.ValueEdge(), index.ValueString()), ctx_->memory); + } + + // lhs is Null + return TypedValue(ctx_->memory); + } + + TypedValue Visit(ListSlicingOperator &op) override { + // If some type is null we can't return null, because throwing exception + // on illegal type has higher priority. + auto is_null = false; + auto get_bound = [&](Expression *bound_expr, int64_t default_value) { + if (bound_expr) { + auto bound = bound_expr->Accept(*this); + if (bound.type() == TypedValue::Type::Null) { + is_null = true; + } else if (bound.type() != TypedValue::Type::Int) { + throw QueryRuntimeException("Expected an integer for a bound in list slicing, got {}.", bound.type()); + } + return bound; + } + return TypedValue(default_value, ctx_->memory); + }; + auto _upper_bound = get_bound(op.upper_bound_, std::numeric_limits::max()); + auto _lower_bound = get_bound(op.lower_bound_, 0); + + auto _list = op.list_->Accept(*this); + if (_list.type() == TypedValue::Type::Null) { + is_null = true; + } else if (_list.type() != TypedValue::Type::List) { + throw QueryRuntimeException("Expected a list to slice, got {}.", _list.type()); + } + + if (is_null) { + return TypedValue(ctx_->memory); + } + const auto &list = _list.ValueList(); + auto normalise_bound = [&](int64_t bound) { + if (bound < 0) { + bound = static_cast(list.size()) + bound; + } + return std::max(static_cast(0), std::min(bound, static_cast(list.size()))); + }; + auto lower_bound = normalise_bound(_lower_bound.ValueInt()); + auto upper_bound = normalise_bound(_upper_bound.ValueInt()); + if (upper_bound <= lower_bound) { + return TypedValue(TypedValue::TVector(ctx_->memory), ctx_->memory); + } + return TypedValue(TypedValue::TVector(list.begin() + lower_bound, list.begin() + upper_bound, ctx_->memory)); + } + + TypedValue Visit(IsNullOperator &is_null) override { + auto value = is_null.expression_->Accept(*this); + return TypedValue(value.IsNull(), ctx_->memory); + } + + TypedValue Visit(PropertyLookup &property_lookup) override { + auto expression_result = property_lookup.expression_->Accept(*this); + auto maybe_date = [this](const auto &date, const auto &prop_name) -> std::optional { + if (prop_name == "year") { + return TypedValue(date.year, ctx_->memory); + } + if (prop_name == "month") { + return TypedValue(date.month, ctx_->memory); + } + if (prop_name == "day") { + return TypedValue(date.day, ctx_->memory); + } + return std::nullopt; + }; + auto maybe_local_time = [this](const auto <, const auto &prop_name) -> std::optional { + if (prop_name == "hour") { + return TypedValue(lt.hour, ctx_->memory); + } + if (prop_name == "minute") { + return TypedValue(lt.minute, ctx_->memory); + } + if (prop_name == "second") { + return TypedValue(lt.second, ctx_->memory); + } + if (prop_name == "millisecond") { + return TypedValue(lt.millisecond, ctx_->memory); + } + if (prop_name == "microsecond") { + return TypedValue(lt.microsecond, ctx_->memory); + } + return std::nullopt; + }; + auto maybe_duration = [this](const auto &dur, const auto &prop_name) -> std::optional { + if (prop_name == "day") { + return TypedValue(dur.Days(), ctx_->memory); + } + if (prop_name == "hour") { + return TypedValue(dur.SubDaysAsHours(), ctx_->memory); + } + if (prop_name == "minute") { + return TypedValue(dur.SubDaysAsMinutes(), ctx_->memory); + } + if (prop_name == "second") { + return TypedValue(dur.SubDaysAsSeconds(), ctx_->memory); + } + if (prop_name == "millisecond") { + return TypedValue(dur.SubDaysAsMilliseconds(), ctx_->memory); + } + if (prop_name == "microsecond") { + return TypedValue(dur.SubDaysAsMicroseconds(), ctx_->memory); + } + if (prop_name == "nanosecond") { + return TypedValue(dur.SubDaysAsNanoseconds(), ctx_->memory); + } + return std::nullopt; + }; + switch (expression_result.type()) { + case TypedValue::Type::Null: + return TypedValue(ctx_->memory); + case TypedValue::Type::Vertex: + return TypedValue(GetProperty(expression_result.ValueVertex(), property_lookup.property_), ctx_->memory); + case TypedValue::Type::Edge: + return TypedValue(GetProperty(expression_result.ValueEdge(), property_lookup.property_), ctx_->memory); + case TypedValue::Type::Map: { + // NOTE: Take non-const reference to map, so that we can move out the + // looked-up element as the result. + auto &map = expression_result.ValueMap(); + auto found = map.find(property_lookup.property_.name.c_str()); + if (found == map.end()) return TypedValue(ctx_->memory); + // NOTE: Explicit move is needed, so that we return the move constructed + // value and preserve the correct MemoryResource. + return std::move(found->second); + } + case TypedValue::Type::Duration: { + const auto &prop_name = property_lookup.property_.name; + const auto &dur = expression_result.ValueDuration(); + if (auto dur_field = maybe_duration(dur, prop_name); dur_field) { + return std::move(*dur_field); + } + throw QueryRuntimeException("Invalid property name {} for Duration", prop_name); + } + case TypedValue::Type::Date: { + const auto &prop_name = property_lookup.property_.name; + const auto &date = expression_result.ValueDate(); + if (auto date_field = maybe_date(date, prop_name); date_field) { + return std::move(*date_field); + } + throw QueryRuntimeException("Invalid property name {} for Date", prop_name); + } + case TypedValue::Type::LocalTime: { + const auto &prop_name = property_lookup.property_.name; + const auto < = expression_result.ValueLocalTime(); + if (auto lt_field = maybe_local_time(lt, prop_name); lt_field) { + return std::move(*lt_field); + } + throw QueryRuntimeException("Invalid property name {} for LocalTime", prop_name); + } + case TypedValue::Type::LocalDateTime: { + const auto &prop_name = property_lookup.property_.name; + const auto &ldt = expression_result.ValueLocalDateTime(); + if (auto date_field = maybe_date(ldt.date, prop_name); date_field) { + return std::move(*date_field); + } + if (auto lt_field = maybe_local_time(ldt.local_time, prop_name); lt_field) { + return std::move(*lt_field); + } + throw QueryRuntimeException("Invalid property name {} for LocalDateTime", prop_name); + } + default: + throw QueryRuntimeException("Only nodes, edges, maps and temporal types have properties to be looked-up."); + } + } + + TypedValue Visit(LabelsTest &labels_test) override { + auto expression_result = labels_test.expression_->Accept(*this); + switch (expression_result.type()) { + case TypedValue::Type::Null: + return TypedValue(ctx_->memory); + case TypedValue::Type::Vertex: { + const auto &vertex = expression_result.ValueVertex(); + for (const auto &label : labels_test.labels_) { + auto has_label = vertex.HasLabel(view_, GetLabel(label)); + if (has_label.HasError() && has_label.GetError() == storage::v3::Error::NONEXISTENT_OBJECT) { + // This is a very nasty and temporary hack in order to make MERGE + // work. The old storage had the following logic when returning an + // `OLD` view: `return old ? old : new`. That means that if the + // `OLD` view didn't exist, it returned the NEW view. With this hack + // we simulate that behavior. + // TODO (mferencevic, teon.banek): Remove once MERGE is + // reimplemented. + has_label = vertex.HasLabel(storage::v3::View::NEW, GetLabel(label)); + } + if (has_label.HasError()) { + switch (has_label.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to access labels on a deleted node."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to access labels from a node that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when accessing labels."); + } + } + if (!*has_label) { + return TypedValue(false, ctx_->memory); + } + } + return TypedValue(true, ctx_->memory); + } + default: + throw QueryRuntimeException("Only nodes have labels."); + } + } + + TypedValue Visit(PrimitiveLiteral &literal) override { + // TODO: no need to evaluate constants, we can write it to frame in one + // of the previous phases. + return TypedValue(literal.value_, ctx_->memory); + } + + TypedValue Visit(ListLiteral &literal) override { + TypedValue::TVector result(ctx_->memory); + result.reserve(literal.elements_.size()); + for (const auto &expression : literal.elements_) result.emplace_back(expression->Accept(*this)); + return TypedValue(result, ctx_->memory); + } + + TypedValue Visit(MapLiteral &literal) override { + TypedValue::TMap result(ctx_->memory); + for (const auto &pair : literal.elements_) result.emplace(pair.first.name, pair.second->Accept(*this)); + return TypedValue(result, ctx_->memory); + } + + TypedValue Visit(Aggregation &aggregation) override { + return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory); + } + + TypedValue Visit(Coalesce &coalesce) override { + auto &exprs = coalesce.expressions_; + + if (exprs.size() == 0) { + throw QueryRuntimeException("'coalesce' requires at least one argument."); + } + + for (int64_t i = 0; i < exprs.size(); ++i) { + TypedValue val(exprs[i]->Accept(*this), ctx_->memory); + if (!val.IsNull()) { + return val; + } + } + + return TypedValue(ctx_->memory); + } + + TypedValue Visit(Function &function) override { + FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_}; + // Stack allocate evaluated arguments when there's a small number of them. + if (function.arguments_.size() <= 8) { + TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory), + TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory), + TypedValue(ctx_->memory), TypedValue(ctx_->memory)}; + for (size_t i = 0; i < function.arguments_.size(); ++i) { + arguments[i] = function.arguments_[i]->Accept(*this); + } + auto res = function.function_(arguments, function.arguments_.size(), function_ctx); + MG_ASSERT(res.GetMemoryResource() == ctx_->memory); + return res; + } else { + TypedValue::TVector arguments(ctx_->memory); + arguments.reserve(function.arguments_.size()); + for (const auto &argument : function.arguments_) { + arguments.emplace_back(argument->Accept(*this)); + } + auto res = function.function_(arguments.data(), arguments.size(), function_ctx); + MG_ASSERT(res.GetMemoryResource() == ctx_->memory); + return res; + } + } + + TypedValue Visit(Reduce &reduce) override { + auto list_value = reduce.list_->Accept(*this); + if (list_value.IsNull()) { + return TypedValue(ctx_->memory); + } + if (list_value.type() != TypedValue::Type::List) { + throw QueryRuntimeException("REDUCE expected a list, got {}.", list_value.type()); + } + const auto &list = list_value.ValueList(); + const auto &element_symbol = symbol_table_->at(*reduce.identifier_); + const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_); + auto accumulator = reduce.initializer_->Accept(*this); + for (const auto &element : list) { + frame_->at(accumulator_symbol) = accumulator; + frame_->at(element_symbol) = element; + accumulator = reduce.expression_->Accept(*this); + } + return accumulator; + } + + TypedValue Visit(Extract &extract) override { + auto list_value = extract.list_->Accept(*this); + if (list_value.IsNull()) { + return TypedValue(ctx_->memory); + } + if (list_value.type() != TypedValue::Type::List) { + throw QueryRuntimeException("EXTRACT expected a list, got {}.", list_value.type()); + } + const auto &list = list_value.ValueList(); + const auto &element_symbol = symbol_table_->at(*extract.identifier_); + TypedValue::TVector result(ctx_->memory); + result.reserve(list.size()); + for (const auto &element : list) { + if (element.IsNull()) { + result.emplace_back(); + } else { + frame_->at(element_symbol) = element; + result.emplace_back(extract.expression_->Accept(*this)); + } + } + return TypedValue(result, ctx_->memory); + } + + TypedValue Visit(All &all) override { + auto list_value = all.list_expression_->Accept(*this); + if (list_value.IsNull()) { + return TypedValue(ctx_->memory); + } + if (list_value.type() != TypedValue::Type::List) { + throw QueryRuntimeException("ALL expected a list, got {}.", list_value.type()); + } + const auto &list = list_value.ValueList(); + const auto &symbol = symbol_table_->at(*all.identifier_); + bool has_null_elements = false; + bool has_value = false; + for (const auto &element : list) { + frame_->at(symbol) = element; + auto result = all.where_->expression_->Accept(*this); + if (!result.IsNull() && result.type() != TypedValue::Type::Bool) { + throw QueryRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type()); + } + if (!result.IsNull()) { + has_value = true; + if (!result.ValueBool()) { + return TypedValue(false, ctx_->memory); + } + } else { + has_null_elements = true; + } + } + if (!has_value) { + return TypedValue(ctx_->memory); + } + if (has_null_elements) { + return TypedValue(false, ctx_->memory); + } else { + return TypedValue(true, ctx_->memory); + } + } + + TypedValue Visit(Single &single) override { + auto list_value = single.list_expression_->Accept(*this); + if (list_value.IsNull()) { + return TypedValue(ctx_->memory); + } + if (list_value.type() != TypedValue::Type::List) { + throw QueryRuntimeException("SINGLE expected a list, got {}.", list_value.type()); + } + const auto &list = list_value.ValueList(); + const auto &symbol = symbol_table_->at(*single.identifier_); + bool has_value = false; + bool predicate_satisfied = false; + for (const auto &element : list) { + frame_->at(symbol) = element; + auto result = single.where_->expression_->Accept(*this); + if (!result.IsNull() && result.type() != TypedValue::Type::Bool) { + throw QueryRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type()); + } + if (result.type() == TypedValue::Type::Bool) { + has_value = true; + } + if (result.IsNull() || !result.ValueBool()) { + continue; + } + // Return false if more than one element satisfies the predicate. + if (predicate_satisfied) { + return TypedValue(false, ctx_->memory); + } else { + predicate_satisfied = true; + } + } + if (!has_value) { + return TypedValue(ctx_->memory); + } else { + return TypedValue(predicate_satisfied, ctx_->memory); + } + } + + TypedValue Visit(Any &any) override { + auto list_value = any.list_expression_->Accept(*this); + if (list_value.IsNull()) { + return TypedValue(ctx_->memory); + } + if (list_value.type() != TypedValue::Type::List) { + throw QueryRuntimeException("ANY expected a list, got {}.", list_value.type()); + } + const auto &list = list_value.ValueList(); + const auto &symbol = symbol_table_->at(*any.identifier_); + bool has_value = false; + for (const auto &element : list) { + frame_->at(symbol) = element; + auto result = any.where_->expression_->Accept(*this); + if (!result.IsNull() && result.type() != TypedValue::Type::Bool) { + throw QueryRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type()); + } + if (!result.IsNull()) { + has_value = true; + if (result.ValueBool()) { + return TypedValue(true, ctx_->memory); + } + } + } + // Return Null if all elements are Null + if (!has_value) { + return TypedValue(ctx_->memory); + } else { + return TypedValue(false, ctx_->memory); + } + } + + TypedValue Visit(None &none) override { + auto list_value = none.list_expression_->Accept(*this); + if (list_value.IsNull()) { + return TypedValue(ctx_->memory); + } + if (list_value.type() != TypedValue::Type::List) { + throw QueryRuntimeException("NONE expected a list, got {}.", list_value.type()); + } + const auto &list = list_value.ValueList(); + const auto &symbol = symbol_table_->at(*none.identifier_); + bool has_value = false; + for (const auto &element : list) { + frame_->at(symbol) = element; + auto result = none.where_->expression_->Accept(*this); + if (!result.IsNull() && result.type() != TypedValue::Type::Bool) { + throw QueryRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type()); + } + if (!result.IsNull()) { + has_value = true; + if (result.ValueBool()) { + return TypedValue(false, ctx_->memory); + } + } + } + // Return Null if all elements are Null + if (!has_value) { + return TypedValue(ctx_->memory); + } else { + return TypedValue(true, ctx_->memory); + } + } + + TypedValue Visit(ParameterLookup ¶m_lookup) override { + return TypedValue(ctx_->parameters.AtTokenPosition(param_lookup.token_position_), ctx_->memory); + } + + TypedValue Visit(RegexMatch ®ex_match) override { + auto target_string_value = regex_match.string_expr_->Accept(*this); + auto regex_value = regex_match.regex_->Accept(*this); + if (target_string_value.IsNull() || regex_value.IsNull()) { + return TypedValue(ctx_->memory); + } + if (regex_value.type() != TypedValue::Type::String) { + throw QueryRuntimeException("Regular expression must evaluate to a string, got {}.", regex_value.type()); + } + if (target_string_value.type() != TypedValue::Type::String) { + // Instead of error, we return Null which makes it compatible in case we + // use indexed lookup which filters out any non-string properties. + // Assuming a property lookup is the target_string_value. + return TypedValue(ctx_->memory); + } + const auto &target_string = target_string_value.ValueString(); + try { + std::regex regex(regex_value.ValueString()); + return TypedValue(std::regex_match(target_string, regex), ctx_->memory); + } catch (const std::regex_error &e) { + throw QueryRuntimeException("Regex error in '{}': {}", regex_value.ValueString(), e.what()); + } + } + + private: + template + storage::v3::PropertyValue GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) { + auto maybe_prop = record_accessor.GetProperty(view_, ctx_->properties[prop.ix]); + if (maybe_prop.HasError() && maybe_prop.GetError() == storage::v3::Error::NONEXISTENT_OBJECT) { + // This is a very nasty and temporary hack in order to make MERGE work. + // The old storage had the following logic when returning an `OLD` view: + // `return old ? old : new`. That means that if the `OLD` view didn't + // exist, it returned the NEW view. With this hack we simulate that + // behavior. + // TODO (mferencevic, teon.banek): Remove once MERGE is reimplemented. + maybe_prop = record_accessor.GetProperty(storage::v3::View::NEW, ctx_->properties[prop.ix]); + } + if (maybe_prop.HasError()) { + switch (maybe_prop.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to get a property from a deleted object."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get a property from an object that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when getting a property."); + } + } + return *maybe_prop; + } + + template + storage::v3::PropertyValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) { + auto maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name)); + if (maybe_prop.HasError() && maybe_prop.GetError() == storage::v3::Error::NONEXISTENT_OBJECT) { + // This is a very nasty and temporary hack in order to make MERGE work. + // The old storage had the following logic when returning an `OLD` view: + // `return old ? old : new`. That means that if the `OLD` view didn't + // exist, it returned the NEW view. With this hack we simulate that + // behavior. + // TODO (mferencevic, teon.banek): Remove once MERGE is reimplemented. + maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name)); + } + if (maybe_prop.HasError()) { + switch (maybe_prop.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to get a property from a deleted object."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get a property from an object that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when getting a property."); + } + } + return *maybe_prop; + } + + storage::v3::LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; } + + Frame *frame_; + const SymbolTable *symbol_table_; + const EvaluationContext *ctx_; + DbAccessor *dba_; + // which switching approach should be used when evaluating + storage::v3::View view_; +}; + +/// A helper function for evaluating an expression that's an int. +/// +/// @param what - Name of what's getting evaluated. Used for user feedback (via +/// exception) when the evaluated value is not an int. +/// @throw QueryRuntimeException if expression doesn't evaluate to an int. +int64_t EvaluateInt(ExpressionEvaluator *evaluator, Expression *expr, const std::string &what); + +std::optional EvaluateMemoryLimit(ExpressionEvaluator *eval, Expression *memory_limit, size_t memory_scale); + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/interpret/frame.hpp b/src/query/v2/interpret/frame.hpp new file mode 100644 index 000000000..6b02a8a6c --- /dev/null +++ b/src/query/v2/interpret/frame.hpp @@ -0,0 +1,45 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "query/v2/frontend/semantic/symbol_table.hpp" +#include "query/v2/typed_value.hpp" +#include "utils/logging.hpp" +#include "utils/memory.hpp" +#include "utils/pmr/vector.hpp" + +namespace memgraph::query::v2 { + +class Frame { + public: + /// Create a Frame of given size backed by a utils::NewDeleteResource() + explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); } + + Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); } + + TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; } + const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; } + + TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); } + const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); } + + auto &elems() { return elems_; } + + utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); } + + private: + utils::pmr::vector elems_; +}; + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/interpreter.cpp b/src/query/v2/interpreter.cpp new file mode 100644 index 000000000..a60d77dec --- /dev/null +++ b/src/query/v2/interpreter.cpp @@ -0,0 +1,2412 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/interpreter.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "memory/memory_control.hpp" +#include "query/v2/constants.hpp" +#include "query/v2/context.hpp" +#include "query/v2/cypher_query_interpreter.hpp" +#include "query/v2/db_accessor.hpp" +#include "query/v2/dump.hpp" +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/ast/ast_visitor.hpp" +#include "query/v2/frontend/ast/cypher_main_visitor.hpp" +#include "query/v2/frontend/opencypher/parser.hpp" +#include "query/v2/frontend/semantic/required_privileges.hpp" +#include "query/v2/frontend/semantic/symbol_generator.hpp" +#include "query/v2/interpret/eval.hpp" +#include "query/v2/metadata.hpp" +#include "query/v2/plan/planner.hpp" +#include "query/v2/plan/profile.hpp" +#include "query/v2/plan/vertex_count_cache.hpp" +#include "query/v2/stream/common.hpp" +#include "query/v2/trigger.hpp" +#include "query/v2/typed_value.hpp" +#include "storage/v3/property_value.hpp" +#include "storage/v3/storage.hpp" +#include "utils/algorithm.hpp" +#include "utils/csv_parsing.hpp" +#include "utils/event_counter.hpp" +#include "utils/exceptions.hpp" +#include "utils/flag_validation.hpp" +#include "utils/license.hpp" +#include "utils/likely.hpp" +#include "utils/logging.hpp" +#include "utils/memory.hpp" +#include "utils/memory_tracker.hpp" +#include "utils/readable_size.hpp" +#include "utils/settings.hpp" +#include "utils/string.hpp" +#include "utils/tsc.hpp" +#include "utils/variant_helpers.hpp" + +namespace EventCounter { +extern Event ReadQuery; +extern Event WriteQuery; +extern Event ReadWriteQuery; + +extern const Event LabelIndexCreated; +extern const Event LabelPropertyIndexCreated; + +extern const Event StreamsCreated; +extern const Event TriggersCreated; +} // namespace EventCounter + +namespace memgraph::query::v2 { + +namespace { +void UpdateTypeCount(const plan::ReadWriteTypeChecker::RWType type) { + switch (type) { + case plan::ReadWriteTypeChecker::RWType::R: + EventCounter::IncrementCounter(EventCounter::ReadQuery); + break; + case plan::ReadWriteTypeChecker::RWType::W: + EventCounter::IncrementCounter(EventCounter::WriteQuery); + break; + case plan::ReadWriteTypeChecker::RWType::RW: + EventCounter::IncrementCounter(EventCounter::ReadWriteQuery); + break; + default: + break; + } +} + +struct Callback { + std::vector header; + using CallbackFunction = std::function>()>; + CallbackFunction fn; + bool should_abort_query{false}; +}; + +TypedValue EvaluateOptionalExpression(Expression *expression, ExpressionEvaluator *eval) { + return expression ? expression->Accept(*eval) : TypedValue(); +} + +template +std::optional GetOptionalValue(query::v2::Expression *expression, ExpressionEvaluator &evaluator) { + if (expression != nullptr) { + auto int_value = expression->Accept(evaluator); + MG_ASSERT(int_value.IsNull() || int_value.IsInt()); + if (int_value.IsInt()) { + return TResult{int_value.ValueInt()}; + } + } + return {}; +}; + +std::optional GetOptionalStringValue(query::v2::Expression *expression, ExpressionEvaluator &evaluator) { + if (expression != nullptr) { + auto value = expression->Accept(evaluator); + MG_ASSERT(value.IsNull() || value.IsString()); + if (value.IsString()) { + return {std::string(value.ValueString().begin(), value.ValueString().end())}; + } + } + return {}; +}; + +class ReplQueryHandler final : public query::v2::ReplicationQueryHandler { + public: + explicit ReplQueryHandler(storage::v3::Storage *db) : db_(db) {} + + /// @throw QueryRuntimeException if an error ocurred. + void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional port) override { + if (replication_role == ReplicationQuery::ReplicationRole::MAIN) { + if (!db_->SetMainReplicationRole()) { + throw QueryRuntimeException("Couldn't set role to main!"); + } + } + if (replication_role == ReplicationQuery::ReplicationRole::REPLICA) { + if (!port || *port < 0 || *port > std::numeric_limits::max()) { + throw QueryRuntimeException("Port number invalid!"); + } + if (!db_->SetReplicaRole( + io::network::Endpoint(query::v2::kDefaultReplicationServerIp, static_cast(*port)))) { + throw QueryRuntimeException("Couldn't set role to replica!"); + } + } + } + + /// @throw QueryRuntimeException if an error ocurred. + ReplicationQuery::ReplicationRole ShowReplicationRole() const override { + switch (db_->GetReplicationRole()) { + case storage::v3::ReplicationRole::MAIN: + return ReplicationQuery::ReplicationRole::MAIN; + case storage::v3::ReplicationRole::REPLICA: + return ReplicationQuery::ReplicationRole::REPLICA; + } + throw QueryRuntimeException("Couldn't show replication role - invalid role set!"); + } + + /// @throw QueryRuntimeException if an error ocurred. + void RegisterReplica(const std::string &name, const std::string &socket_address, + const ReplicationQuery::SyncMode sync_mode, const std::optional timeout, + const std::chrono::seconds replica_check_frequency) override { + if (db_->GetReplicationRole() == storage::v3::ReplicationRole::REPLICA) { + // replica can't register another replica + throw QueryRuntimeException("Replica can't register another replica!"); + } + + storage::v3::replication::ReplicationMode repl_mode; + switch (sync_mode) { + case ReplicationQuery::SyncMode::ASYNC: { + repl_mode = storage::v3::replication::ReplicationMode::ASYNC; + break; + } + case ReplicationQuery::SyncMode::SYNC: { + repl_mode = storage::v3::replication::ReplicationMode::SYNC; + break; + } + } + + auto maybe_ip_and_port = + io::network::Endpoint::ParseSocketOrIpAddress(socket_address, query::v2::kDefaultReplicationPort); + if (maybe_ip_and_port) { + auto [ip, port] = *maybe_ip_and_port; + auto ret = db_->RegisterReplica( + name, {std::move(ip), port}, repl_mode, + {.timeout = timeout, .replica_check_frequency = replica_check_frequency, .ssl = std::nullopt}); + if (ret.HasError()) { + throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name)); + } + } else { + throw QueryRuntimeException("Invalid socket address!"); + } + } + + /// @throw QueryRuntimeException if an error ocurred. + void DropReplica(const std::string &replica_name) override { + if (db_->GetReplicationRole() == storage::v3::ReplicationRole::REPLICA) { + // replica can't unregister a replica + throw QueryRuntimeException("Replica can't unregister a replica!"); + } + if (!db_->UnregisterReplica(replica_name)) { + throw QueryRuntimeException(fmt::format("Couldn't unregister the replica '{}'", replica_name)); + } + } + + using Replica = ReplicationQueryHandler::Replica; + std::vector ShowReplicas() const override { + if (db_->GetReplicationRole() == storage::v3::ReplicationRole::REPLICA) { + // replica can't show registered replicas (it shouldn't have any) + throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!"); + } + + auto repl_infos = db_->ReplicasInfo(); + std::vector replicas; + replicas.reserve(repl_infos.size()); + + const auto from_info = [](const auto &repl_info) -> Replica { + Replica replica; + replica.name = repl_info.name; + replica.socket_address = repl_info.endpoint.SocketAddress(); + switch (repl_info.mode) { + case storage::v3::replication::ReplicationMode::SYNC: + replica.sync_mode = ReplicationQuery::SyncMode::SYNC; + break; + case storage::v3::replication::ReplicationMode::ASYNC: + replica.sync_mode = ReplicationQuery::SyncMode::ASYNC; + break; + } + if (repl_info.timeout) { + replica.timeout = *repl_info.timeout; + } + + switch (repl_info.state) { + case storage::v3::replication::ReplicaState::READY: + replica.state = ReplicationQuery::ReplicaState::READY; + break; + case storage::v3::replication::ReplicaState::REPLICATING: + replica.state = ReplicationQuery::ReplicaState::REPLICATING; + break; + case storage::v3::replication::ReplicaState::RECOVERY: + replica.state = ReplicationQuery::ReplicaState::RECOVERY; + break; + case storage::v3::replication::ReplicaState::INVALID: + replica.state = ReplicationQuery::ReplicaState::INVALID; + break; + } + + return replica; + }; + + std::transform(repl_infos.begin(), repl_infos.end(), std::back_inserter(replicas), from_info); + return replicas; + } + + private: + storage::v3::Storage *db_; +}; +/// returns false if the replication role can't be set +/// @throw QueryRuntimeException if an error ocurred. + +Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Parameters ¶meters, + DbAccessor *db_accessor) { + // Empty frame for evaluation of password expression. This is OK since + // password should be either null or string literal and it's evaluation + // should not depend on frame. + Frame frame(0); + SymbolTable symbol_table; + EvaluationContext evaluation_context; + // TODO: MemoryResource for EvaluationContext, it should probably be passed as + // the argument to Callback. + evaluation_context.timestamp = QueryTimestamp(); + evaluation_context.parameters = parameters; + ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD); + + std::string username = auth_query->user_; + std::string rolename = auth_query->role_; + std::string user_or_role = auth_query->user_or_role_; + std::vector privileges = auth_query->privileges_; + auto password = EvaluateOptionalExpression(auth_query->password_, &evaluator); + + Callback callback; + + const auto license_check_result = utils::license::global_license_checker.IsValidLicense(utils::global_settings); + + static const std::unordered_set enterprise_only_methods{ + AuthQuery::Action::CREATE_ROLE, AuthQuery::Action::DROP_ROLE, AuthQuery::Action::SET_ROLE, + AuthQuery::Action::CLEAR_ROLE, AuthQuery::Action::GRANT_PRIVILEGE, AuthQuery::Action::DENY_PRIVILEGE, + AuthQuery::Action::REVOKE_PRIVILEGE, AuthQuery::Action::SHOW_PRIVILEGES, AuthQuery::Action::SHOW_USERS_FOR_ROLE, + AuthQuery::Action::SHOW_ROLE_FOR_USER}; + + if (license_check_result.HasError() && enterprise_only_methods.contains(auth_query->action_)) { + throw utils::BasicException( + utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "advanced authentication features")); + } + + switch (auth_query->action_) { + case AuthQuery::Action::CREATE_USER: + callback.fn = [auth, username, password, valid_enterprise_license = !license_check_result.HasError()] { + MG_ASSERT(password.IsString() || password.IsNull()); + if (!auth->CreateUser(username, password.IsString() ? std::make_optional(std::string(password.ValueString())) + : std::nullopt)) { + throw QueryRuntimeException("User '{}' already exists.", username); + } + + // If the license is not valid we create users with admin access + if (!valid_enterprise_license) { + spdlog::warn("Granting all the privileges to {}.", username); + auth->GrantPrivilege(username, kPrivilegesAll); + } + + return std::vector>(); + }; + return callback; + case AuthQuery::Action::DROP_USER: + callback.fn = [auth, username] { + if (!auth->DropUser(username)) { + throw QueryRuntimeException("User '{}' doesn't exist.", username); + } + return std::vector>(); + }; + return callback; + case AuthQuery::Action::SET_PASSWORD: + callback.fn = [auth, username, password] { + MG_ASSERT(password.IsString() || password.IsNull()); + auth->SetPassword(username, + password.IsString() ? std::make_optional(std::string(password.ValueString())) : std::nullopt); + return std::vector>(); + }; + return callback; + case AuthQuery::Action::CREATE_ROLE: + callback.fn = [auth, rolename] { + if (!auth->CreateRole(rolename)) { + throw QueryRuntimeException("Role '{}' already exists.", rolename); + } + return std::vector>(); + }; + return callback; + case AuthQuery::Action::DROP_ROLE: + callback.fn = [auth, rolename] { + if (!auth->DropRole(rolename)) { + throw QueryRuntimeException("Role '{}' doesn't exist.", rolename); + } + return std::vector>(); + }; + return callback; + case AuthQuery::Action::SHOW_USERS: + callback.header = {"user"}; + callback.fn = [auth] { + std::vector> rows; + auto usernames = auth->GetUsernames(); + rows.reserve(usernames.size()); + for (auto &&username : usernames) { + rows.emplace_back(std::vector{username}); + } + return rows; + }; + return callback; + case AuthQuery::Action::SHOW_ROLES: + callback.header = {"role"}; + callback.fn = [auth] { + std::vector> rows; + auto rolenames = auth->GetRolenames(); + rows.reserve(rolenames.size()); + for (auto &&rolename : rolenames) { + rows.emplace_back(std::vector{rolename}); + } + return rows; + }; + return callback; + case AuthQuery::Action::SET_ROLE: + callback.fn = [auth, username, rolename] { + auth->SetRole(username, rolename); + return std::vector>(); + }; + return callback; + case AuthQuery::Action::CLEAR_ROLE: + callback.fn = [auth, username] { + auth->ClearRole(username); + return std::vector>(); + }; + return callback; + case AuthQuery::Action::GRANT_PRIVILEGE: + callback.fn = [auth, user_or_role, privileges] { + auth->GrantPrivilege(user_or_role, privileges); + return std::vector>(); + }; + return callback; + case AuthQuery::Action::DENY_PRIVILEGE: + callback.fn = [auth, user_or_role, privileges] { + auth->DenyPrivilege(user_or_role, privileges); + return std::vector>(); + }; + return callback; + case AuthQuery::Action::REVOKE_PRIVILEGE: { + callback.fn = [auth, user_or_role, privileges] { + auth->RevokePrivilege(user_or_role, privileges); + return std::vector>(); + }; + return callback; + } + case AuthQuery::Action::SHOW_PRIVILEGES: + callback.header = {"privilege", "effective", "description"}; + callback.fn = [auth, user_or_role] { return auth->GetPrivileges(user_or_role); }; + return callback; + case AuthQuery::Action::SHOW_ROLE_FOR_USER: + callback.header = {"role"}; + callback.fn = [auth, username] { + auto maybe_rolename = auth->GetRolenameForUser(username); + return std::vector>{ + std::vector{TypedValue(maybe_rolename ? *maybe_rolename : "null")}}; + }; + return callback; + case AuthQuery::Action::SHOW_USERS_FOR_ROLE: + callback.header = {"users"}; + callback.fn = [auth, rolename] { + std::vector> rows; + auto usernames = auth->GetUsernamesForRole(rolename); + rows.reserve(usernames.size()); + for (auto &&username : usernames) { + rows.emplace_back(std::vector{username}); + } + return rows; + }; + return callback; + default: + break; + } +} + +Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters ¶meters, + InterpreterContext *interpreter_context, DbAccessor *db_accessor, + std::vector *notifications) { + Frame frame(0); + SymbolTable symbol_table; + EvaluationContext evaluation_context; + // TODO: MemoryResource for EvaluationContext, it should probably be passed as + // the argument to Callback. + evaluation_context.timestamp = QueryTimestamp(); + evaluation_context.parameters = parameters; + ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD); + + Callback callback; + switch (repl_query->action_) { + case ReplicationQuery::Action::SET_REPLICATION_ROLE: { + auto port = EvaluateOptionalExpression(repl_query->port_, &evaluator); + std::optional maybe_port; + if (port.IsInt()) { + maybe_port = port.ValueInt(); + } + if (maybe_port == 7687 && repl_query->role_ == ReplicationQuery::ReplicationRole::REPLICA) { + notifications->emplace_back(SeverityLevel::WARNING, NotificationCode::REPLICA_PORT_WARNING, + "Be careful the replication port must be different from the memgraph port!"); + } + callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, role = repl_query->role_, + maybe_port]() mutable { + handler.SetReplicationRole(role, maybe_port); + return std::vector>(); + }; + notifications->emplace_back( + SeverityLevel::INFO, NotificationCode::SET_REPLICA, + fmt::format("Replica role set to {}.", + repl_query->role_ == ReplicationQuery::ReplicationRole::MAIN ? "MAIN" : "REPLICA")); + return callback; + } + case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: { + callback.header = {"replication role"}; + callback.fn = [handler = ReplQueryHandler{interpreter_context->db}] { + auto mode = handler.ShowReplicationRole(); + switch (mode) { + case ReplicationQuery::ReplicationRole::MAIN: { + return std::vector>{{TypedValue("main")}}; + } + case ReplicationQuery::ReplicationRole::REPLICA: { + return std::vector>{{TypedValue("replica")}}; + } + } + }; + return callback; + } + case ReplicationQuery::Action::REGISTER_REPLICA: { + const auto &name = repl_query->replica_name_; + const auto &sync_mode = repl_query->sync_mode_; + auto socket_address = repl_query->socket_address_->Accept(evaluator); + auto timeout = EvaluateOptionalExpression(repl_query->timeout_, &evaluator); + const auto replica_check_frequency = interpreter_context->config.replication_replica_check_frequency; + std::optional maybe_timeout; + if (timeout.IsDouble()) { + maybe_timeout = timeout.ValueDouble(); + } else if (timeout.IsInt()) { + maybe_timeout = static_cast(timeout.ValueInt()); + } + callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, name, socket_address, sync_mode, + maybe_timeout, replica_check_frequency]() mutable { + handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, maybe_timeout, + replica_check_frequency); + return std::vector>(); + }; + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::REGISTER_REPLICA, + fmt::format("Replica {} is registered.", repl_query->replica_name_)); + return callback; + } + + case ReplicationQuery::Action::DROP_REPLICA: { + const auto &name = repl_query->replica_name_; + callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, name]() mutable { + handler.DropReplica(name); + return std::vector>(); + }; + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::DROP_REPLICA, + fmt::format("Replica {} is dropped.", repl_query->replica_name_)); + return callback; + } + + case ReplicationQuery::Action::SHOW_REPLICAS: { + callback.header = {"name", "socket_address", "sync_mode", "timeout", "state"}; + callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, replica_nfields = callback.header.size()] { + const auto &replicas = handler.ShowReplicas(); + auto typed_replicas = std::vector>{}; + typed_replicas.reserve(replicas.size()); + for (const auto &replica : replicas) { + std::vector typed_replica; + typed_replica.reserve(replica_nfields); + + typed_replica.emplace_back(TypedValue(replica.name)); + typed_replica.emplace_back(TypedValue(replica.socket_address)); + + switch (replica.sync_mode) { + case ReplicationQuery::SyncMode::SYNC: + typed_replica.emplace_back(TypedValue("sync")); + break; + case ReplicationQuery::SyncMode::ASYNC: + typed_replica.emplace_back(TypedValue("async")); + break; + } + + if (replica.timeout) { + typed_replica.emplace_back(TypedValue(*replica.timeout)); + } else { + typed_replica.emplace_back(TypedValue()); + } + + switch (replica.state) { + case ReplicationQuery::ReplicaState::READY: + typed_replica.emplace_back(TypedValue("ready")); + break; + case ReplicationQuery::ReplicaState::REPLICATING: + typed_replica.emplace_back(TypedValue("replicating")); + break; + case ReplicationQuery::ReplicaState::RECOVERY: + typed_replica.emplace_back(TypedValue("recovery")); + break; + case ReplicationQuery::ReplicaState::INVALID: + typed_replica.emplace_back(TypedValue("invalid")); + break; + } + + typed_replicas.emplace_back(std::move(typed_replica)); + } + return typed_replicas; + }; + return callback; + } + } +} + +std::optional StringPointerToOptional(const std::string *str) { + return str == nullptr ? std::nullopt : std::make_optional(*str); +} + +stream::CommonStreamInfo GetCommonStreamInfo(StreamQuery *stream_query, ExpressionEvaluator &evaluator) { + return { + .batch_interval = GetOptionalValue(stream_query->batch_interval_, evaluator) + .value_or(stream::kDefaultBatchInterval), + .batch_size = GetOptionalValue(stream_query->batch_size_, evaluator).value_or(stream::kDefaultBatchSize), + .transformation_name = stream_query->transform_name_}; +} + +std::vector EvaluateTopicNames(ExpressionEvaluator &evaluator, + std::variant> topic_variant) { + return std::visit(utils::Overloaded{[&](Expression *expression) { + auto topic_names = expression->Accept(evaluator); + MG_ASSERT(topic_names.IsString()); + return utils::Split(topic_names.ValueString(), ","); + }, + [&](std::vector topic_names) { return topic_names; }}, + std::move(topic_variant)); +} + +Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, ExpressionEvaluator &evaluator, + InterpreterContext *interpreter_context, + const std::string *username) { + static constexpr std::string_view kDefaultConsumerGroup = "mg_consumer"; + std::string consumer_group{stream_query->consumer_group_.empty() ? kDefaultConsumerGroup + : stream_query->consumer_group_}; + + auto bootstrap = GetOptionalStringValue(stream_query->bootstrap_servers_, evaluator); + if (bootstrap && bootstrap->empty()) { + throw SemanticException("Bootstrap servers must not be an empty string!"); + } + auto common_stream_info = GetCommonStreamInfo(stream_query, evaluator); + + const auto get_config_map = [&evaluator](std::unordered_map map, + std::string_view map_name) -> std::unordered_map { + std::unordered_map config_map; + for (const auto [key_expr, value_expr] : map) { + const auto key = key_expr->Accept(evaluator); + const auto value = value_expr->Accept(evaluator); + if (!key.IsString() || !value.IsString()) { + throw SemanticException("{} must contain only string keys and values!", map_name); + } + config_map.emplace(key.ValueString(), value.ValueString()); + } + return config_map; + }; + + return [interpreter_context, stream_name = stream_query->stream_name_, + topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_), + consumer_group = std::move(consumer_group), common_stream_info = std::move(common_stream_info), + bootstrap_servers = std::move(bootstrap), owner = StringPointerToOptional(username), + configs = get_config_map(stream_query->configs_, "Configs"), + credentials = get_config_map(stream_query->credentials_, "Credentials")]() mutable { + std::string bootstrap = bootstrap_servers + ? std::move(*bootstrap_servers) + : std::string{interpreter_context->config.default_kafka_bootstrap_servers}; + interpreter_context->streams.Create(stream_name, + {.common_info = std::move(common_stream_info), + .topics = std::move(topic_names), + .consumer_group = std::move(consumer_group), + .bootstrap_servers = std::move(bootstrap), + .configs = std::move(configs), + .credentials = std::move(credentials)}, + std::move(owner)); + + return std::vector>{}; + }; +} + +Callback::CallbackFunction GetPulsarCreateCallback(StreamQuery *stream_query, ExpressionEvaluator &evaluator, + InterpreterContext *interpreter_context, + const std::string *username) { + auto service_url = GetOptionalStringValue(stream_query->service_url_, evaluator); + if (service_url && service_url->empty()) { + throw SemanticException("Service URL must not be an empty string!"); + } + auto common_stream_info = GetCommonStreamInfo(stream_query, evaluator); + return [interpreter_context, stream_name = stream_query->stream_name_, + topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_), + common_stream_info = std::move(common_stream_info), service_url = std::move(service_url), + owner = StringPointerToOptional(username)]() mutable { + std::string url = + service_url ? std::move(*service_url) : std::string{interpreter_context->config.default_pulsar_service_url}; + interpreter_context->streams.Create( + stream_name, + {.common_info = std::move(common_stream_info), .topics = std::move(topic_names), .service_url = std::move(url)}, + std::move(owner)); + + return std::vector>{}; + }; +} + +Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters ¶meters, + InterpreterContext *interpreter_context, DbAccessor *db_accessor, + const std::string *username, std::vector *notifications) { + Frame frame(0); + SymbolTable symbol_table; + EvaluationContext evaluation_context; + // TODO: MemoryResource for EvaluationContext, it should probably be passed as + // the argument to Callback. + evaluation_context.timestamp = QueryTimestamp(); + evaluation_context.parameters = parameters; + ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD); + + Callback callback; + switch (stream_query->action_) { + case StreamQuery::Action::CREATE_STREAM: { + EventCounter::IncrementCounter(EventCounter::StreamsCreated); + switch (stream_query->type_) { + case StreamQuery::Type::KAFKA: + callback.fn = GetKafkaCreateCallback(stream_query, evaluator, interpreter_context, username); + break; + case StreamQuery::Type::PULSAR: + callback.fn = GetPulsarCreateCallback(stream_query, evaluator, interpreter_context, username); + break; + } + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::CREATE_STREAM, + fmt::format("Created stream {}.", stream_query->stream_name_)); + return callback; + } + case StreamQuery::Action::START_STREAM: { + const auto batch_limit = GetOptionalValue(stream_query->batch_limit_, evaluator); + const auto timeout = GetOptionalValue(stream_query->timeout_, evaluator); + + if (batch_limit.has_value()) { + if (batch_limit.value() < 0) { + throw utils::BasicException("Parameter BATCH_LIMIT cannot hold negative value"); + } + + callback.fn = [interpreter_context, stream_name = stream_query->stream_name_, batch_limit, timeout]() { + interpreter_context->streams.StartWithLimit(stream_name, static_cast(batch_limit.value()), timeout); + return std::vector>{}; + }; + } else { + callback.fn = [interpreter_context, stream_name = stream_query->stream_name_]() { + interpreter_context->streams.Start(stream_name); + return std::vector>{}; + }; + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::START_STREAM, + fmt::format("Started stream {}.", stream_query->stream_name_)); + } + return callback; + } + case StreamQuery::Action::START_ALL_STREAMS: { + callback.fn = [interpreter_context]() { + interpreter_context->streams.StartAll(); + return std::vector>{}; + }; + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::START_ALL_STREAMS, "Started all streams."); + return callback; + } + case StreamQuery::Action::STOP_STREAM: { + callback.fn = [interpreter_context, stream_name = stream_query->stream_name_]() { + interpreter_context->streams.Stop(stream_name); + return std::vector>{}; + }; + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::STOP_STREAM, + fmt::format("Stopped stream {}.", stream_query->stream_name_)); + return callback; + } + case StreamQuery::Action::STOP_ALL_STREAMS: { + callback.fn = [interpreter_context]() { + interpreter_context->streams.StopAll(); + return std::vector>{}; + }; + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::STOP_ALL_STREAMS, "Stopped all streams."); + return callback; + } + case StreamQuery::Action::DROP_STREAM: { + callback.fn = [interpreter_context, stream_name = stream_query->stream_name_]() { + interpreter_context->streams.Drop(stream_name); + return std::vector>{}; + }; + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::DROP_STREAM, + fmt::format("Dropped stream {}.", stream_query->stream_name_)); + return callback; + } + case StreamQuery::Action::SHOW_STREAMS: { + callback.header = {"name", "type", "batch_interval", "batch_size", "transformation_name", "owner", "is running"}; + callback.fn = [interpreter_context]() { + auto streams_status = interpreter_context->streams.GetStreamInfo(); + std::vector> results; + results.reserve(streams_status.size()); + auto stream_info_as_typed_stream_info_emplace_in = [](auto &typed_status, const auto &stream_info) { + typed_status.emplace_back(stream_info.batch_interval.count()); + typed_status.emplace_back(stream_info.batch_size); + typed_status.emplace_back(stream_info.transformation_name); + }; + + for (const auto &status : streams_status) { + std::vector typed_status; + typed_status.reserve(7); + typed_status.emplace_back(status.name); + typed_status.emplace_back(StreamSourceTypeToString(status.type)); + stream_info_as_typed_stream_info_emplace_in(typed_status, status.info); + if (status.owner.has_value()) { + typed_status.emplace_back(*status.owner); + } else { + typed_status.emplace_back(); + } + typed_status.emplace_back(status.is_running); + results.push_back(std::move(typed_status)); + } + + return results; + }; + return callback; + } + case StreamQuery::Action::CHECK_STREAM: { + callback.header = {"queries", "raw messages"}; + + const auto batch_limit = GetOptionalValue(stream_query->batch_limit_, evaluator); + if (batch_limit.has_value() && batch_limit.value() < 0) { + throw utils::BasicException("Parameter BATCH_LIMIT cannot hold negative value"); + } + + callback.fn = [interpreter_context, stream_name = stream_query->stream_name_, + timeout = GetOptionalValue(stream_query->timeout_, evaluator), + batch_limit]() mutable { + return interpreter_context->streams.Check(stream_name, timeout, batch_limit); + }; + notifications->emplace_back(SeverityLevel::INFO, NotificationCode::CHECK_STREAM, + fmt::format("Checked stream {}.", stream_query->stream_name_)); + return callback; + } + } +} + +Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters ¶meters, DbAccessor *db_accessor) { + Frame frame(0); + SymbolTable symbol_table; + EvaluationContext evaluation_context; + // TODO: MemoryResource for EvaluationContext, it should probably be passed as + // the argument to Callback. + evaluation_context.timestamp = + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count(); + evaluation_context.parameters = parameters; + ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD); + + Callback callback; + switch (setting_query->action_) { + case SettingQuery::Action::SET_SETTING: { + const auto setting_name = EvaluateOptionalExpression(setting_query->setting_name_, &evaluator); + if (!setting_name.IsString()) { + throw utils::BasicException("Setting name should be a string literal"); + } + + const auto setting_value = EvaluateOptionalExpression(setting_query->setting_value_, &evaluator); + if (!setting_value.IsString()) { + throw utils::BasicException("Setting value should be a string literal"); + } + + callback.fn = [setting_name = std::string{setting_name.ValueString()}, + setting_value = std::string{setting_value.ValueString()}]() mutable { + if (!utils::global_settings.SetValue(setting_name, setting_value)) { + throw utils::BasicException("Unknown setting name '{}'", setting_name); + } + return std::vector>{}; + }; + return callback; + } + case SettingQuery::Action::SHOW_SETTING: { + const auto setting_name = EvaluateOptionalExpression(setting_query->setting_name_, &evaluator); + if (!setting_name.IsString()) { + throw utils::BasicException("Setting name should be a string literal"); + } + + callback.header = {"setting_value"}; + callback.fn = [setting_name = std::string{setting_name.ValueString()}] { + auto maybe_value = utils::global_settings.GetValue(setting_name); + if (!maybe_value) { + throw utils::BasicException("Unknown setting name '{}'", setting_name); + } + std::vector> results; + results.reserve(1); + + std::vector setting_value; + setting_value.reserve(1); + + setting_value.emplace_back(*maybe_value); + results.push_back(std::move(setting_value)); + return results; + }; + return callback; + } + case SettingQuery::Action::SHOW_ALL_SETTINGS: { + callback.header = {"setting_name", "setting_value"}; + callback.fn = [] { + auto all_settings = utils::global_settings.AllSettings(); + std::vector> results; + results.reserve(all_settings.size()); + + for (const auto &[k, v] : all_settings) { + std::vector setting_info; + setting_info.reserve(2); + + setting_info.emplace_back(k); + setting_info.emplace_back(v); + results.push_back(std::move(setting_info)); + } + + return results; + }; + return callback; + } + } +} + +// Struct for lazy pulling from a vector +struct PullPlanVector { + explicit PullPlanVector(std::vector> values) : values_(std::move(values)) {} + + // @return true if there are more unstreamed elements in vector, + // false otherwise. + bool Pull(AnyStream *stream, std::optional n) { + int local_counter{0}; + while (global_counter < values_.size() && (!n || local_counter < n)) { + stream->Result(values_[global_counter]); + ++global_counter; + ++local_counter; + } + + return global_counter == values_.size(); + } + + private: + int global_counter{0}; + std::vector> values_; +}; + +struct PullPlan { + explicit PullPlan(std::shared_ptr plan, const Parameters ¶meters, bool is_profile_query, + DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory, + TriggerContextCollector *trigger_context_collector = nullptr, + std::optional memory_limit = {}); + std::optional Pull(AnyStream *stream, std::optional n, + const std::vector &output_symbols, + std::map *summary); + + private: + std::shared_ptr plan_ = nullptr; + plan::UniqueCursorPtr cursor_ = nullptr; + Frame frame_; + ExecutionContext ctx_; + std::optional memory_limit_; + + // As it's possible to query execution using multiple pulls + // we need the keep track of the total execution time across + // those pulls by accumulating the execution time. + std::chrono::duration execution_time_{0}; + + // To pull the results from a query we call the `Pull` method on + // the cursor which saves the results in a Frame. + // Becuase we can't find out if there are some saved results in a frame, + // and the cursor cannot deduce if the next pull will have a result, + // we have to keep track of any unsent results from previous `PullPlan::Pull` + // manually by using this flag. + bool has_unsent_results_ = false; +}; + +PullPlan::PullPlan(const std::shared_ptr plan, const Parameters ¶meters, const bool is_profile_query, + DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory, + TriggerContextCollector *trigger_context_collector, const std::optional memory_limit) + : plan_(plan), + cursor_(plan->plan().MakeCursor(execution_memory)), + frame_(plan->symbol_table().max_position(), execution_memory), + memory_limit_(memory_limit) { + ctx_.db_accessor = dba; + ctx_.symbol_table = plan->symbol_table(); + ctx_.evaluation_context.timestamp = QueryTimestamp(); + ctx_.evaluation_context.parameters = parameters; + ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, dba); + ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, dba); + if (interpreter_context->config.execution_timeout_sec > 0) { + ctx_.timer = utils::AsyncTimer{interpreter_context->config.execution_timeout_sec}; + } + ctx_.is_shutting_down = &interpreter_context->is_shutting_down; + ctx_.is_profile_query = is_profile_query; + ctx_.trigger_context_collector = trigger_context_collector; +} + +std::optional PullPlan::Pull(AnyStream *stream, std::optional n, + const std::vector &output_symbols, + std::map *summary) { + // Set up temporary memory for a single Pull. Initial memory comes from the + // stack. 256 KiB should fit on the stack and should be more than enough for a + // single `Pull`. + static constexpr size_t stack_size = 256UL * 1024UL; + char stack_data[stack_size]; + utils::ResourceWithOutOfMemoryException resource_with_exception; + utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception); + // We can throw on every query because a simple queries for deleting will use only + // the stack allocated buffer. + // Also, we want to throw only when the query engine requests more memory and not the storage + // so we add the exception to the allocator. + // TODO (mferencevic): Tune the parameters accordingly. + utils::PoolResource pool_memory(128, 1024, &monotonic_memory); + std::optional maybe_limited_resource; + + if (memory_limit_) { + maybe_limited_resource.emplace(&pool_memory, *memory_limit_); + ctx_.evaluation_context.memory = &*maybe_limited_resource; + } else { + ctx_.evaluation_context.memory = &pool_memory; + } + + // Returns true if a result was pulled. + const auto pull_result = [&]() -> bool { return cursor_->Pull(frame_, ctx_); }; + + const auto stream_values = [&]() { + // TODO: The streamed values should also probably use the above memory. + std::vector values; + values.reserve(output_symbols.size()); + + for (const auto &symbol : output_symbols) { + values.emplace_back(frame_[symbol]); + } + + stream->Result(values); + }; + + // Get the execution time of all possible result pulls and streams. + utils::Timer timer; + + int i = 0; + if (has_unsent_results_ && !output_symbols.empty()) { + // stream unsent results from previous pull + stream_values(); + ++i; + } + + for (; !n || i < n; ++i) { + if (!pull_result()) { + break; + } + + if (!output_symbols.empty()) { + stream_values(); + } + } + + // If we finished because we streamed the requested n results, + // we try to pull the next result to see if there is more. + // If there is additional result, we leave the pulled result in the frame + // and set the flag to true. + has_unsent_results_ = i == n && pull_result(); + + execution_time_ += timer.Elapsed(); + + if (has_unsent_results_) { + return std::nullopt; + } + summary->insert_or_assign("plan_execution_time", execution_time_.count()); + // We are finished with pulling all the data, therefore we can send any + // metadata about the results i.e. notifications and statistics + const bool is_any_counter_set = + std::any_of(ctx_.execution_stats.counters.begin(), ctx_.execution_stats.counters.end(), + [](const auto &counter) { return counter > 0; }); + if (is_any_counter_set) { + std::map stats; + for (size_t i = 0; i < ctx_.execution_stats.counters.size(); ++i) { + stats.emplace(ExecutionStatsKeyToString(ExecutionStats::Key(i)), ctx_.execution_stats.counters[i]); + } + summary->insert_or_assign("stats", std::move(stats)); + } + cursor_->Shutdown(); + ctx_.profile_execution_time = execution_time_; + return GetStatsWithTotalTime(ctx_); +} + +using RWType = plan::ReadWriteTypeChecker::RWType; +} // namespace + +InterpreterContext::InterpreterContext(storage::v3::Storage *db, const InterpreterConfig config, + const std::filesystem::path &data_directory) + : db(db), trigger_store(data_directory / "triggers"), config(config), streams{this, data_directory / "streams"} {} + +Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) { + MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL"); +} + +PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper) { + std::function handler; + + if (query_upper == "BEGIN") { + handler = [this] { + if (in_explicit_transaction_) { + throw ExplicitTransactionUsageException("Nested transactions are not supported."); + } + in_explicit_transaction_ = true; + expect_rollback_ = false; + + db_accessor_ = std::make_unique( + interpreter_context_->db->Access(GetIsolationLevelOverride())); + execution_db_accessor_.emplace(db_accessor_.get()); + + if (interpreter_context_->trigger_store.HasTriggers()) { + trigger_context_collector_.emplace(interpreter_context_->trigger_store.GetEventTypes()); + } + }; + } else if (query_upper == "COMMIT") { + handler = [this] { + if (!in_explicit_transaction_) { + throw ExplicitTransactionUsageException("No current transaction to commit."); + } + if (expect_rollback_) { + throw ExplicitTransactionUsageException( + "Transaction can't be committed because there was a previous " + "error. Please invoke a rollback instead."); + } + + try { + Commit(); + } catch (const utils::BasicException &) { + AbortCommand(nullptr); + throw; + } + + expect_rollback_ = false; + in_explicit_transaction_ = false; + }; + } else if (query_upper == "ROLLBACK") { + handler = [this] { + if (!in_explicit_transaction_) { + throw ExplicitTransactionUsageException("No current transaction to rollback."); + } + Abort(); + expect_rollback_ = false; + in_explicit_transaction_ = false; + }; + } else { + LOG_FATAL("Should not get here -- unknown transaction query!"); + } + + return {{}, + {}, + [handler = std::move(handler)](AnyStream *, std::optional) { + handler(); + return QueryHandlerResult::NOTHING; + }, + RWType::NONE}; +} + +PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map *summary, + InterpreterContext *interpreter_context, DbAccessor *dba, + utils::MemoryResource *execution_memory, std::vector *notifications, + TriggerContextCollector *trigger_context_collector = nullptr) { + auto *cypher_query = utils::Downcast(parsed_query.query); + + Frame frame(0); + SymbolTable symbol_table; + EvaluationContext evaluation_context; + evaluation_context.timestamp = QueryTimestamp(); + evaluation_context.parameters = parsed_query.parameters; + ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::v3::View::OLD); + const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_); + if (memory_limit) { + spdlog::info("Running query with memory limit of {}", utils::GetReadableSize(*memory_limit)); + } + + if (const auto &clauses = cypher_query->single_query_->clauses_; std::any_of( + clauses.begin(), clauses.end(), [](const auto *clause) { return clause->GetTypeInfo() == LoadCsv::kType; })) { + notifications->emplace_back( + SeverityLevel::INFO, NotificationCode::LOAD_CSV_TIP, + "It's important to note that the parser parses the values as strings. It's up to the user to " + "convert the parsed row values to the appropriate type. This can be done using the built-in " + "conversion functions such as ToInteger, ToFloat, ToBoolean etc."); + } + + auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query, + parsed_query.parameters, + parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba); + + summary->insert_or_assign("cost_estimate", plan->cost()); + auto rw_type_checker = plan::ReadWriteTypeChecker(); + rw_type_checker.InferRWType(const_cast(plan->plan())); + + auto output_symbols = plan->plan().OutputSymbols(plan->symbol_table()); + + std::vector header; + header.reserve(output_symbols.size()); + + for (const auto &symbol : output_symbols) { + // When the symbol is aliased or expanded from '*' (inside RETURN or + // WITH), then there is no token position, so use symbol name. + // Otherwise, find the name from stripped query. + header.push_back( + utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first); + } + auto pull_plan = std::make_shared(plan, parsed_query.parameters, false, dba, interpreter_context, + execution_memory, trigger_context_collector, memory_limit); + return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), + [pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary]( + AnyStream *stream, std::optional n) -> std::optional { + if (pull_plan->Pull(stream, n, output_symbols, summary)) { + return QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + rw_type_checker.type}; +} + +PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map *summary, + InterpreterContext *interpreter_context, DbAccessor *dba, + utils::MemoryResource *execution_memory) { + const std::string kExplainQueryStart = "explain "; + MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart), + "Expected stripped query to start with '{}'", kExplainQueryStart); + + // Parse and cache the inner query separately (as if it was a standalone + // query), producing a fresh AST. Note that currently we cannot just reuse + // part of the already produced AST because the parameters within ASTs are + // looked up using their positions within the string that was parsed. These + // wouldn't match up if if we were to reuse the AST (produced by parsing the + // full query string) when given just the inner query to execute. + ParsedQuery parsed_inner_query = + ParseQuery(parsed_query.query_string.substr(kExplainQueryStart.size()), parsed_query.user_parameters, + &interpreter_context->ast_cache, &interpreter_context->antlr_lock, interpreter_context->config.query); + + auto *cypher_query = utils::Downcast(parsed_inner_query.query); + MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN"); + + auto cypher_query_plan = CypherQueryToPlan( + parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query, + parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba); + + std::stringstream printed_plan; + plan::PrettyPrint(*dba, &cypher_query_plan->plan(), &printed_plan); + + std::vector> printed_plan_rows; + for (const auto &row : utils::Split(utils::RTrim(printed_plan.str()), "\n")) { + printed_plan_rows.push_back(std::vector{TypedValue(row)}); + } + + summary->insert_or_assign("explain", plan::PlanToJson(*dba, &cypher_query_plan->plan()).dump()); + + return PreparedQuery{{"QUERY PLAN"}, + std::move(parsed_query.required_privileges), + [pull_plan = std::make_shared(std::move(printed_plan_rows))]( + AnyStream *stream, std::optional n) -> std::optional { + if (pull_plan->Pull(stream, n)) { + return QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + RWType::NONE}; +} + +PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction, + std::map *summary, InterpreterContext *interpreter_context, + DbAccessor *dba, utils::MemoryResource *execution_memory) { + const std::string kProfileQueryStart = "profile "; + + MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart), + "Expected stripped query to start with '{}'", kProfileQueryStart); + + // PROFILE isn't allowed inside multi-command (explicit) transactions. This is + // because PROFILE executes each PROFILE'd query and collects additional + // perfomance metadata that it displays to the user instead of the results + // yielded by the query. Because PROFILE has side-effects, each transaction + // that is used to execute a PROFILE query *MUST* be aborted. That isn't + // possible when using multicommand (explicit) transactions (because the user + // controls the lifetime of the transaction) and that is why PROFILE is + // explicitly disabled here in multicommand (explicit) transactions. + // NOTE: Unlike PROFILE, EXPLAIN doesn't have any unwanted side-effects (in + // transaction terms) because it doesn't execute the query, it just prints its + // query plan. That is why EXPLAIN can be used in multicommand (explicit) + // transactions. + if (in_explicit_transaction) { + throw ProfileInMulticommandTxException(); + } + + if (!interpreter_context->tsc_frequency) { + throw QueryException("TSC support is missing for PROFILE"); + } + + // Parse and cache the inner query separately (as if it was a standalone + // query), producing a fresh AST. Note that currently we cannot just reuse + // part of the already produced AST because the parameters within ASTs are + // looked up using their positions within the string that was parsed. These + // wouldn't match up if if we were to reuse the AST (produced by parsing the + // full query string) when given just the inner query to execute. + ParsedQuery parsed_inner_query = + ParseQuery(parsed_query.query_string.substr(kProfileQueryStart.size()), parsed_query.user_parameters, + &interpreter_context->ast_cache, &interpreter_context->antlr_lock, interpreter_context->config.query); + + auto *cypher_query = utils::Downcast(parsed_inner_query.query); + MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE"); + Frame frame(0); + SymbolTable symbol_table; + EvaluationContext evaluation_context; + evaluation_context.timestamp = QueryTimestamp(); + evaluation_context.parameters = parsed_inner_query.parameters; + ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::v3::View::OLD); + const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_); + + auto cypher_query_plan = CypherQueryToPlan( + parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query, + parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba); + auto rw_type_checker = plan::ReadWriteTypeChecker(); + rw_type_checker.InferRWType(const_cast(cypher_query_plan->plan())); + + return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"}, + std::move(parsed_query.required_privileges), + [plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), + summary, dba, interpreter_context, execution_memory, memory_limit, + // We want to execute the query we are profiling lazily, so we delay + // the construction of the corresponding context. + stats_and_total_time = std::optional{}, + pull_plan = std::shared_ptr(nullptr)]( + AnyStream *stream, std::optional n) mutable -> std::optional { + // No output symbols are given so that nothing is streamed. + if (!stats_and_total_time) { + stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context, + execution_memory, nullptr, memory_limit) + .Pull(stream, {}, {}, summary); + pull_plan = std::make_shared(ProfilingStatsToTable(*stats_and_total_time)); + } + + MG_ASSERT(stats_and_total_time, "Failed to execute the query!"); + + if (pull_plan->Pull(stream, n)) { + summary->insert_or_assign("profile", ProfilingStatsToJson(*stats_and_total_time).dump()); + return QueryHandlerResult::ABORT; + } + + return std::nullopt; + }, + rw_type_checker.type}; +} + +PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map *summary, DbAccessor *dba, + utils::MemoryResource *execution_memory) { + return PreparedQuery{{"QUERY"}, + std::move(parsed_query.required_privileges), + [pull_plan = std::make_shared(dba)]( + AnyStream *stream, std::optional n) -> std::optional { + if (pull_plan->Pull(stream, n)) { + return QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + RWType::R}; +} + +PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction, + std::vector *notifications, InterpreterContext *interpreter_context) { + if (in_explicit_transaction) { + throw IndexInMulticommandTxException(); + } + + auto *index_query = utils::Downcast(parsed_query.query); + std::function handler; + + // Creating an index influences computed plan costs. + auto invalidate_plan_cache = [plan_cache = &interpreter_context->plan_cache] { + auto access = plan_cache->access(); + for (auto &kv : access) { + access.remove(kv.first); + } + }; + + auto label = interpreter_context->db->NameToLabel(index_query->label_.name); + + std::vector properties; + std::vector properties_string; + properties.reserve(index_query->properties_.size()); + properties_string.reserve(index_query->properties_.size()); + for (const auto &prop : index_query->properties_) { + properties.push_back(interpreter_context->db->NameToProperty(prop.name)); + properties_string.push_back(prop.name); + } + auto properties_stringified = utils::Join(properties_string, ", "); + + if (properties.size() > 1) { + throw utils::NotYetImplemented("index on multiple properties"); + } + + Notification index_notification(SeverityLevel::INFO); + switch (index_query->action_) { + case IndexQuery::Action::CREATE: { + index_notification.code = NotificationCode::CREATE_INDEX; + index_notification.title = + fmt::format("Created index on label {} on properties {}.", index_query->label_.name, properties_stringified); + + handler = [interpreter_context, label, properties_stringified = std::move(properties_stringified), + label_name = index_query->label_.name, properties = std::move(properties), + invalidate_plan_cache = std::move(invalidate_plan_cache)](Notification &index_notification) { + if (properties.empty()) { + if (!interpreter_context->db->CreateIndex(label)) { + index_notification.code = NotificationCode::EXISTANT_INDEX; + index_notification.title = + fmt::format("Index on label {} on properties {} already exists.", label_name, properties_stringified); + } + EventCounter::IncrementCounter(EventCounter::LabelIndexCreated); + } else { + MG_ASSERT(properties.size() == 1U); + if (!interpreter_context->db->CreateIndex(label, properties[0])) { + index_notification.code = NotificationCode::EXISTANT_INDEX; + index_notification.title = + fmt::format("Index on label {} on properties {} already exists.", label_name, properties_stringified); + } + EventCounter::IncrementCounter(EventCounter::LabelPropertyIndexCreated); + } + invalidate_plan_cache(); + }; + break; + } + case IndexQuery::Action::DROP: { + index_notification.code = NotificationCode::DROP_INDEX; + index_notification.title = fmt::format("Dropped index on label {} on properties {}.", index_query->label_.name, + utils::Join(properties_string, ", ")); + handler = [interpreter_context, label, properties_stringified = std::move(properties_stringified), + label_name = index_query->label_.name, properties = std::move(properties), + invalidate_plan_cache = std::move(invalidate_plan_cache)](Notification &index_notification) { + if (properties.empty()) { + if (!interpreter_context->db->DropIndex(label)) { + index_notification.code = NotificationCode::NONEXISTANT_INDEX; + index_notification.title = + fmt::format("Index on label {} on properties {} doesn't exist.", label_name, properties_stringified); + } + } else { + MG_ASSERT(properties.size() == 1U); + if (!interpreter_context->db->DropIndex(label, properties[0])) { + index_notification.code = NotificationCode::NONEXISTANT_INDEX; + index_notification.title = + fmt::format("Index on label {} on properties {} doesn't exist.", label_name, properties_stringified); + } + } + invalidate_plan_cache(); + }; + break; + } + } + + return PreparedQuery{ + {}, + std::move(parsed_query.required_privileges), + [handler = std::move(handler), notifications, index_notification = std::move(index_notification)]( + AnyStream * /*stream*/, std::optional /*unused*/) mutable { + handler(index_notification); + notifications->push_back(index_notification); + return QueryHandlerResult::NOTHING; + }, + RWType::W}; +} + +PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction, + std::map *summary, InterpreterContext *interpreter_context, + DbAccessor *dba, utils::MemoryResource *execution_memory) { + if (in_explicit_transaction) { + throw UserModificationInMulticommandTxException(); + } + + auto *auth_query = utils::Downcast(parsed_query.query); + + auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, dba); + + SymbolTable symbol_table; + std::vector output_symbols; + for (const auto &column : callback.header) { + output_symbols.emplace_back(symbol_table.CreateSymbol(column, "false")); + } + + auto plan = std::make_shared(std::make_unique( + std::make_unique(output_symbols, + [fn = callback.fn](Frame *, ExecutionContext *) { return fn(); }), + 0.0, AstStorage{}, symbol_table)); + + auto pull_plan = + std::make_shared(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory); + return PreparedQuery{ + callback.header, std::move(parsed_query.required_privileges), + [pull_plan = std::move(pull_plan), callback = std::move(callback), output_symbols = std::move(output_symbols), + summary](AnyStream *stream, std::optional n) -> std::optional { + if (pull_plan->Pull(stream, n, output_symbols, summary)) { + return callback.should_abort_query ? QueryHandlerResult::ABORT : QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + RWType::NONE}; +} + +PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, + std::vector *notifications, InterpreterContext *interpreter_context, + DbAccessor *dba) { + if (in_explicit_transaction) { + throw ReplicationModificationInMulticommandTxException(); + } + + auto *replication_query = utils::Downcast(parsed_query.query); + auto callback = + HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context, dba, notifications); + + return PreparedQuery{callback.header, std::move(parsed_query.required_privileges), + [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr{nullptr}]( + AnyStream *stream, std::optional n) mutable -> std::optional { + if (UNLIKELY(!pull_plan)) { + pull_plan = std::make_shared(callback_fn()); + } + + if (pull_plan->Pull(stream, n)) { + return QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + RWType::NONE}; + // False positive report for the std::make_shared above + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) +} + +PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, + InterpreterContext *interpreter_context, DbAccessor *dba) { + if (in_explicit_transaction) { + throw LockPathModificationInMulticommandTxException(); + } + + auto *lock_path_query = utils::Downcast(parsed_query.query); + + return PreparedQuery{{}, + std::move(parsed_query.required_privileges), + [interpreter_context, action = lock_path_query->action_]( + AnyStream *stream, std::optional n) -> std::optional { + switch (action) { + case LockPathQuery::Action::LOCK_PATH: + if (!interpreter_context->db->LockPath()) { + throw QueryRuntimeException("Failed to lock the data directory"); + } + break; + case LockPathQuery::Action::UNLOCK_PATH: + if (!interpreter_context->db->UnlockPath()) { + throw QueryRuntimeException("Failed to unlock the data directory"); + } + break; + } + return QueryHandlerResult::COMMIT; + }, + RWType::NONE}; +} + +PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, + InterpreterContext *interpreter_context) { + if (in_explicit_transaction) { + throw FreeMemoryModificationInMulticommandTxException(); + } + + return PreparedQuery{ + {}, + std::move(parsed_query.required_privileges), + [interpreter_context](AnyStream *stream, std::optional n) -> std::optional { + interpreter_context->db->FreeMemory(); + memory::PurgeUnusedMemory(); + return QueryHandlerResult::COMMIT; + }, + RWType::NONE}; +} + +TriggerEventType ToTriggerEventType(const TriggerQuery::EventType event_type) { + switch (event_type) { + case TriggerQuery::EventType::ANY: + return TriggerEventType::ANY; + + case TriggerQuery::EventType::CREATE: + return TriggerEventType::CREATE; + + case TriggerQuery::EventType::VERTEX_CREATE: + return TriggerEventType::VERTEX_CREATE; + + case TriggerQuery::EventType::EDGE_CREATE: + return TriggerEventType::EDGE_CREATE; + + case TriggerQuery::EventType::DELETE: + return TriggerEventType::DELETE; + + case TriggerQuery::EventType::VERTEX_DELETE: + return TriggerEventType::VERTEX_DELETE; + + case TriggerQuery::EventType::EDGE_DELETE: + return TriggerEventType::EDGE_DELETE; + + case TriggerQuery::EventType::UPDATE: + return TriggerEventType::UPDATE; + + case TriggerQuery::EventType::VERTEX_UPDATE: + return TriggerEventType::VERTEX_UPDATE; + + case TriggerQuery::EventType::EDGE_UPDATE: + return TriggerEventType::EDGE_UPDATE; + } +} + +Callback CreateTrigger(TriggerQuery *trigger_query, + const std::map &user_parameters, + InterpreterContext *interpreter_context, DbAccessor *dba, std::optional owner) { + return { + {}, + [trigger_name = std::move(trigger_query->trigger_name_), trigger_statement = std::move(trigger_query->statement_), + event_type = trigger_query->event_type_, before_commit = trigger_query->before_commit_, interpreter_context, dba, + user_parameters, owner = std::move(owner)]() mutable -> std::vector> { + interpreter_context->trigger_store.AddTrigger( + std::move(trigger_name), trigger_statement, user_parameters, ToTriggerEventType(event_type), + before_commit ? TriggerPhase::BEFORE_COMMIT : TriggerPhase::AFTER_COMMIT, &interpreter_context->ast_cache, + dba, &interpreter_context->antlr_lock, interpreter_context->config.query, std::move(owner), + interpreter_context->auth_checker); + return {}; + }}; +} + +Callback DropTrigger(TriggerQuery *trigger_query, InterpreterContext *interpreter_context) { + return {{}, + [trigger_name = std::move(trigger_query->trigger_name_), + interpreter_context]() -> std::vector> { + interpreter_context->trigger_store.DropTrigger(trigger_name); + return {}; + }}; +} + +Callback ShowTriggers(InterpreterContext *interpreter_context) { + return {{"trigger name", "statement", "event type", "phase", "owner"}, [interpreter_context] { + std::vector> results; + auto trigger_infos = interpreter_context->trigger_store.GetTriggerInfo(); + results.reserve(trigger_infos.size()); + for (auto &trigger_info : trigger_infos) { + std::vector typed_trigger_info; + typed_trigger_info.reserve(4); + typed_trigger_info.emplace_back(std::move(trigger_info.name)); + typed_trigger_info.emplace_back(std::move(trigger_info.statement)); + typed_trigger_info.emplace_back(TriggerEventTypeToString(trigger_info.event_type)); + typed_trigger_info.emplace_back(trigger_info.phase == TriggerPhase::BEFORE_COMMIT ? "BEFORE COMMIT" + : "AFTER COMMIT"); + typed_trigger_info.emplace_back(trigger_info.owner.has_value() ? TypedValue{*trigger_info.owner} + : TypedValue{}); + + results.push_back(std::move(typed_trigger_info)); + } + + return results; + }}; +} + +PreparedQuery PrepareTriggerQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, + std::vector *notifications, InterpreterContext *interpreter_context, + DbAccessor *dba, + const std::map &user_parameters, + const std::string *username) { + if (in_explicit_transaction) { + throw TriggerModificationInMulticommandTxException(); + } + + auto *trigger_query = utils::Downcast(parsed_query.query); + MG_ASSERT(trigger_query); + + std::optional trigger_notification; + auto callback = std::invoke([trigger_query, interpreter_context, dba, &user_parameters, + owner = StringPointerToOptional(username), &trigger_notification]() mutable { + switch (trigger_query->action_) { + case TriggerQuery::Action::CREATE_TRIGGER: + trigger_notification.emplace(SeverityLevel::INFO, NotificationCode::CREATE_TRIGGER, + fmt::format("Created trigger {}.", trigger_query->trigger_name_)); + EventCounter::IncrementCounter(EventCounter::TriggersCreated); + return CreateTrigger(trigger_query, user_parameters, interpreter_context, dba, std::move(owner)); + case TriggerQuery::Action::DROP_TRIGGER: + trigger_notification.emplace(SeverityLevel::INFO, NotificationCode::DROP_TRIGGER, + fmt::format("Dropped trigger {}.", trigger_query->trigger_name_)); + return DropTrigger(trigger_query, interpreter_context); + case TriggerQuery::Action::SHOW_TRIGGERS: + return ShowTriggers(interpreter_context); + } + }); + + return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges), + [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr{nullptr}, + trigger_notification = std::move(trigger_notification), notifications]( + AnyStream *stream, std::optional n) mutable -> std::optional { + if (UNLIKELY(!pull_plan)) { + pull_plan = std::make_shared(callback_fn()); + } + + if (pull_plan->Pull(stream, n)) { + if (trigger_notification) { + notifications->push_back(std::move(*trigger_notification)); + } + return QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + RWType::NONE}; + // False positive report for the std::make_shared above + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) +} + +PreparedQuery PrepareStreamQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, + std::vector *notifications, InterpreterContext *interpreter_context, + DbAccessor *dba, + const std::map & /*user_parameters*/, + const std::string *username) { + if (in_explicit_transaction) { + throw StreamQueryInMulticommandTxException(); + } + + auto *stream_query = utils::Downcast(parsed_query.query); + MG_ASSERT(stream_query); + auto callback = + HandleStreamQuery(stream_query, parsed_query.parameters, interpreter_context, dba, username, notifications); + + return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges), + [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr{nullptr}]( + AnyStream *stream, std::optional n) mutable -> std::optional { + if (UNLIKELY(!pull_plan)) { + pull_plan = std::make_shared(callback_fn()); + } + + if (pull_plan->Pull(stream, n)) { + return QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + RWType::NONE}; + // False positive report for the std::make_shared above + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) +} + +constexpr auto ToStorageIsolationLevel(const IsolationLevelQuery::IsolationLevel isolation_level) noexcept { + switch (isolation_level) { + case IsolationLevelQuery::IsolationLevel::SNAPSHOT_ISOLATION: + return storage::v3::IsolationLevel::SNAPSHOT_ISOLATION; + case IsolationLevelQuery::IsolationLevel::READ_COMMITTED: + return storage::v3::IsolationLevel::READ_COMMITTED; + case IsolationLevelQuery::IsolationLevel::READ_UNCOMMITTED: + return storage::v3::IsolationLevel::READ_UNCOMMITTED; + } +} + +PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, + InterpreterContext *interpreter_context, Interpreter *interpreter) { + if (in_explicit_transaction) { + throw IsolationLevelModificationInMulticommandTxException(); + } + + auto *isolation_level_query = utils::Downcast(parsed_query.query); + MG_ASSERT(isolation_level_query); + + const auto isolation_level = ToStorageIsolationLevel(isolation_level_query->isolation_level_); + + auto callback = [isolation_level_query, isolation_level, interpreter_context, + interpreter]() -> std::function { + switch (isolation_level_query->isolation_level_scope_) { + case IsolationLevelQuery::IsolationLevelScope::GLOBAL: + return [interpreter_context, isolation_level] { interpreter_context->db->SetIsolationLevel(isolation_level); }; + case IsolationLevelQuery::IsolationLevelScope::SESSION: + return [interpreter, isolation_level] { interpreter->SetSessionIsolationLevel(isolation_level); }; + case IsolationLevelQuery::IsolationLevelScope::NEXT: + return [interpreter, isolation_level] { interpreter->SetNextTransactionIsolationLevel(isolation_level); }; + } + }(); + + return PreparedQuery{ + {}, + std::move(parsed_query.required_privileges), + [callback = std::move(callback)](AnyStream *stream, std::optional n) -> std::optional { + callback(); + return QueryHandlerResult::COMMIT; + }, + RWType::NONE}; +} + +PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_explicit_transaction, + InterpreterContext *interpreter_context) { + if (in_explicit_transaction) { + throw CreateSnapshotInMulticommandTxException(); + } + + return PreparedQuery{ + {}, + std::move(parsed_query.required_privileges), + [interpreter_context](AnyStream *stream, std::optional n) -> std::optional { + if (auto maybe_error = interpreter_context->db->CreateSnapshot(); maybe_error.HasError()) { + switch (maybe_error.GetError()) { + case storage::v3::Storage::CreateSnapshotError::DisabledForReplica: + throw utils::BasicException( + "Failed to create a snapshot. Replica instances are not allowed to create them."); + } + } + return QueryHandlerResult::COMMIT; + }, + RWType::NONE}; +} + +PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, DbAccessor *dba) { + if (in_explicit_transaction) { + throw SettingConfigInMulticommandTxException{}; + } + + auto *setting_query = utils::Downcast(parsed_query.query); + MG_ASSERT(setting_query); + auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, dba); + + return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges), + [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr{nullptr}]( + AnyStream *stream, std::optional n) mutable -> std::optional { + if (UNLIKELY(!pull_plan)) { + pull_plan = std::make_shared(callback_fn()); + } + + if (pull_plan->Pull(stream, n)) { + return QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + RWType::NONE}; + // False positive report for the std::make_shared above + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) +} + +PreparedQuery PrepareVersionQuery(ParsedQuery parsed_query, const bool in_explicit_transaction) { + if (in_explicit_transaction) { + throw VersionInfoInMulticommandTxException(); + } + + return PreparedQuery{{"version"}, + std::move(parsed_query.required_privileges), + [](AnyStream *stream, std::optional /*n*/) { + std::vector version_value; + version_value.reserve(1); + + version_value.emplace_back(gflags::VersionString()); + stream->Result(version_value); + return QueryHandlerResult::COMMIT; + }, + RWType::NONE}; +} + +PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction, + std::map *summary, InterpreterContext *interpreter_context, + storage::v3::Storage *db, utils::MemoryResource *execution_memory) { + if (in_explicit_transaction) { + throw InfoInMulticommandTxException(); + } + + auto *info_query = utils::Downcast(parsed_query.query); + std::vector header; + std::function>, QueryHandlerResult>()> handler; + + switch (info_query->info_type_) { + case InfoQuery::InfoType::STORAGE: + header = {"storage info", "value"}; + handler = [db] { + auto info = db->GetInfo(); + std::vector> results{ + {TypedValue("vertex_count"), TypedValue(static_cast(info.vertex_count))}, + {TypedValue("edge_count"), TypedValue(static_cast(info.edge_count))}, + {TypedValue("average_degree"), TypedValue(info.average_degree)}, + {TypedValue("memory_usage"), TypedValue(static_cast(info.memory_usage))}, + {TypedValue("disk_usage"), TypedValue(static_cast(info.disk_usage))}, + {TypedValue("memory_allocated"), TypedValue(static_cast(utils::total_memory_tracker.Amount()))}, + {TypedValue("allocation_limit"), + TypedValue(static_cast(utils::total_memory_tracker.HardLimit()))}}; + return std::pair{results, QueryHandlerResult::COMMIT}; + }; + break; + case InfoQuery::InfoType::INDEX: + header = {"index type", "label", "property"}; + handler = [interpreter_context] { + auto *db = interpreter_context->db; + auto info = db->ListAllIndices(); + std::vector> results; + results.reserve(info.label.size() + info.label_property.size()); + for (const auto &item : info.label) { + results.push_back({TypedValue("label"), TypedValue(db->LabelToName(item)), TypedValue()}); + } + for (const auto &item : info.label_property) { + results.push_back({TypedValue("label+property"), TypedValue(db->LabelToName(item.first)), + TypedValue(db->PropertyToName(item.second))}); + } + return std::pair{results, QueryHandlerResult::NOTHING}; + }; + break; + case InfoQuery::InfoType::CONSTRAINT: + header = {"constraint type", "label", "properties"}; + handler = [interpreter_context] { + auto *db = interpreter_context->db; + auto info = db->ListAllConstraints(); + std::vector> results; + results.reserve(info.existence.size() + info.unique.size()); + for (const auto &item : info.existence) { + results.push_back({TypedValue("exists"), TypedValue(db->LabelToName(item.first)), + TypedValue(db->PropertyToName(item.second))}); + } + for (const auto &item : info.unique) { + std::vector properties; + properties.reserve(item.second.size()); + for (const auto &property : item.second) { + properties.emplace_back(db->PropertyToName(property)); + } + results.push_back( + {TypedValue("unique"), TypedValue(db->LabelToName(item.first)), TypedValue(std::move(properties))}); + } + return std::pair{results, QueryHandlerResult::NOTHING}; + }; + break; + } + + return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), + [handler = std::move(handler), action = QueryHandlerResult::NOTHING, + pull_plan = std::shared_ptr(nullptr)]( + AnyStream *stream, std::optional n) mutable -> std::optional { + if (!pull_plan) { + auto [results, action_on_complete] = handler(); + action = action_on_complete; + pull_plan = std::make_shared(std::move(results)); + } + + if (pull_plan->Pull(stream, n)) { + return action; + } + return std::nullopt; + }, + RWType::NONE}; +} + +PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_transaction, + std::vector *notifications, + InterpreterContext *interpreter_context) { + if (in_explicit_transaction) { + throw ConstraintInMulticommandTxException(); + } + + auto *constraint_query = utils::Downcast(parsed_query.query); + std::function handler; + + auto label = interpreter_context->db->NameToLabel(constraint_query->constraint_.label.name); + std::vector properties; + std::vector properties_string; + properties.reserve(constraint_query->constraint_.properties.size()); + properties_string.reserve(constraint_query->constraint_.properties.size()); + for (const auto &prop : constraint_query->constraint_.properties) { + properties.push_back(interpreter_context->db->NameToProperty(prop.name)); + properties_string.push_back(prop.name); + } + auto properties_stringified = utils::Join(properties_string, ", "); + + Notification constraint_notification(SeverityLevel::INFO); + switch (constraint_query->action_type_) { + case ConstraintQuery::ActionType::CREATE: { + constraint_notification.code = NotificationCode::CREATE_CONSTRAINT; + + switch (constraint_query->constraint_.type) { + case Constraint::Type::NODE_KEY: + throw utils::NotYetImplemented("Node key constraints"); + case Constraint::Type::EXISTS: + if (properties.empty() || properties.size() > 1) { + throw SyntaxException("Exactly one property must be used for existence constraints."); + } + constraint_notification.title = fmt::format("Created EXISTS constraint on label {} on properties {}.", + constraint_query->constraint_.label.name, properties_stringified); + handler = [interpreter_context, label, label_name = constraint_query->constraint_.label.name, + properties_stringified = std::move(properties_stringified), + properties = std::move(properties)](Notification &constraint_notification) { + auto res = interpreter_context->db->CreateExistenceConstraint(label, properties[0]); + if (res.HasError()) { + auto violation = res.GetError(); + auto label_name = interpreter_context->db->LabelToName(violation.label); + MG_ASSERT(violation.properties.size() == 1U); + auto property_name = interpreter_context->db->PropertyToName(*violation.properties.begin()); + throw QueryRuntimeException( + "Unable to create existence constraint :{}({}), because an " + "existing node violates it.", + label_name, property_name); + } + if (res.HasValue() && !res.GetValue()) { + constraint_notification.code = NotificationCode::EXISTANT_CONSTRAINT; + constraint_notification.title = fmt::format( + "Constraint EXISTS on label {} on properties {} already exists.", label_name, properties_stringified); + } + }; + break; + case Constraint::Type::UNIQUE: + std::set property_set; + for (const auto &property : properties) { + property_set.insert(property); + } + if (property_set.size() != properties.size()) { + throw SyntaxException("The given set of properties contains duplicates."); + } + constraint_notification.title = + fmt::format("Created UNIQUE constraint on label {} on properties {}.", + constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); + handler = [interpreter_context, label, label_name = constraint_query->constraint_.label.name, + properties_stringified = std::move(properties_stringified), + property_set = std::move(property_set)](Notification &constraint_notification) { + auto res = interpreter_context->db->CreateUniqueConstraint(label, property_set); + if (res.HasError()) { + auto violation = res.GetError(); + auto label_name = interpreter_context->db->LabelToName(violation.label); + std::stringstream property_names_stream; + utils::PrintIterable(property_names_stream, violation.properties, ", ", + [&interpreter_context](auto &stream, const auto &prop) { + stream << interpreter_context->db->PropertyToName(prop); + }); + throw QueryRuntimeException( + "Unable to create unique constraint :{}({}), because an " + "existing node violates it.", + label_name, property_names_stream.str()); + } + switch (res.GetValue()) { + case storage::v3::UniqueConstraints::CreationStatus::EMPTY_PROPERTIES: + throw SyntaxException( + "At least one property must be used for unique " + "constraints."); + case storage::v3::UniqueConstraints::CreationStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED: + throw SyntaxException( + "Too many properties specified. Limit of {} properties " + "for unique constraints is exceeded.", + storage::v3::kUniqueConstraintsMaxProperties); + case storage::v3::UniqueConstraints::CreationStatus::ALREADY_EXISTS: + constraint_notification.code = NotificationCode::EXISTANT_CONSTRAINT; + constraint_notification.title = + fmt::format("Constraint UNIQUE on label {} on properties {} already exists.", label_name, + properties_stringified); + break; + case storage::v3::UniqueConstraints::CreationStatus::SUCCESS: + break; + } + }; + break; + } + } break; + case ConstraintQuery::ActionType::DROP: { + constraint_notification.code = NotificationCode::DROP_CONSTRAINT; + + switch (constraint_query->constraint_.type) { + case Constraint::Type::NODE_KEY: + throw utils::NotYetImplemented("Node key constraints"); + case Constraint::Type::EXISTS: + if (properties.empty() || properties.size() > 1) { + throw SyntaxException("Exactly one property must be used for existence constraints."); + } + constraint_notification.title = + fmt::format("Dropped EXISTS constraint on label {} on properties {}.", + constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); + handler = [interpreter_context, label, label_name = constraint_query->constraint_.label.name, + properties_stringified = std::move(properties_stringified), + properties = std::move(properties)](Notification &constraint_notification) { + if (!interpreter_context->db->DropExistenceConstraint(label, properties[0])) { + constraint_notification.code = NotificationCode::NONEXISTANT_CONSTRAINT; + constraint_notification.title = fmt::format( + "Constraint EXISTS on label {} on properties {} doesn't exist.", label_name, properties_stringified); + } + return std::vector>(); + }; + break; + case Constraint::Type::UNIQUE: + std::set property_set; + for (const auto &property : properties) { + property_set.insert(property); + } + if (property_set.size() != properties.size()) { + throw SyntaxException("The given set of properties contains duplicates."); + } + constraint_notification.title = + fmt::format("Dropped UNIQUE constraint on label {} on properties {}.", + constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); + handler = [interpreter_context, label, label_name = constraint_query->constraint_.label.name, + properties_stringified = std::move(properties_stringified), + property_set = std::move(property_set)](Notification &constraint_notification) { + auto res = interpreter_context->db->DropUniqueConstraint(label, property_set); + switch (res) { + case storage::v3::UniqueConstraints::DeletionStatus::EMPTY_PROPERTIES: + throw SyntaxException( + "At least one property must be used for unique " + "constraints."); + break; + case storage::v3::UniqueConstraints::DeletionStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED: + throw SyntaxException( + "Too many properties specified. Limit of {} properties for " + "unique constraints is exceeded.", + storage::v3::kUniqueConstraintsMaxProperties); + break; + case storage::v3::UniqueConstraints::DeletionStatus::NOT_FOUND: + constraint_notification.code = NotificationCode::NONEXISTANT_CONSTRAINT; + constraint_notification.title = + fmt::format("Constraint UNIQUE on label {} on properties {} doesn't exist.", label_name, + properties_stringified); + break; + case storage::v3::UniqueConstraints::DeletionStatus::SUCCESS: + break; + } + return std::vector>(); + }; + } + } break; + } + + return PreparedQuery{{}, + std::move(parsed_query.required_privileges), + [handler = std::move(handler), constraint_notification = std::move(constraint_notification), + notifications](AnyStream * /*stream*/, std::optional /*n*/) mutable { + handler(constraint_notification); + notifications->push_back(constraint_notification); + return QueryHandlerResult::COMMIT; + }, + RWType::NONE}; +} + +void Interpreter::BeginTransaction() { + const auto prepared_query = PrepareTransactionQuery("BEGIN"); + prepared_query.query_handler(nullptr, {}); +} + +void Interpreter::CommitTransaction() { + const auto prepared_query = PrepareTransactionQuery("COMMIT"); + prepared_query.query_handler(nullptr, {}); + query_executions_.clear(); +} + +void Interpreter::RollbackTransaction() { + const auto prepared_query = PrepareTransactionQuery("ROLLBACK"); + prepared_query.query_handler(nullptr, {}); + query_executions_.clear(); +} + +Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, + const std::map ¶ms, + const std::string *username) { + if (!in_explicit_transaction_) { + query_executions_.clear(); + } + + query_executions_.emplace_back(std::make_unique()); + auto &query_execution = query_executions_.back(); + std::optional qid = + in_explicit_transaction_ ? static_cast(query_executions_.size() - 1) : std::optional{}; + + // Handle transaction control queries. + + const auto upper_case_query = utils::ToUpperCase(query_string); + const auto trimmed_query = utils::Trim(upper_case_query); + + if (trimmed_query == "BEGIN" || trimmed_query == "COMMIT" || trimmed_query == "ROLLBACK") { + query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query)); + return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid}; + } + + // All queries other than transaction control queries advance the command in + // an explicit transaction block. + if (in_explicit_transaction_) { + AdvanceCommand(); + } + // If we're not in an explicit transaction block and we have an open + // transaction, abort it since we're about to prepare a new query. + else if (db_accessor_) { + AbortCommand(&query_execution); + } + + try { + // Set a default cost estimate of 0. Individual queries can overwrite this + // field with an improved estimate. + query_execution->summary["cost_estimate"] = 0.0; + + utils::Timer parsing_timer; + ParsedQuery parsed_query = ParseQuery(query_string, params, &interpreter_context_->ast_cache, + &interpreter_context_->antlr_lock, interpreter_context_->config.query); + query_execution->summary["parsing_time"] = parsing_timer.Elapsed().count(); + + // Some queries require an active transaction in order to be prepared. + if (!in_explicit_transaction_ && + (utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || + utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || + utils::Downcast(parsed_query.query))) { + db_accessor_ = std::make_unique( + interpreter_context_->db->Access(GetIsolationLevelOverride())); + execution_db_accessor_.emplace(db_accessor_.get()); + + if (utils::Downcast(parsed_query.query) && interpreter_context_->trigger_store.HasTriggers()) { + trigger_context_collector_.emplace(interpreter_context_->trigger_store.GetEventTypes()); + } + } + + utils::Timer planning_timer; + PreparedQuery prepared_query; + + if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_, + &*execution_db_accessor_, &query_execution->execution_memory, + &query_execution->notifications, + trigger_context_collector_ ? &*trigger_context_collector_ : nullptr); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_, + &*execution_db_accessor_, &query_execution->execution_memory_with_exception); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, + interpreter_context_, &*execution_db_accessor_, + &query_execution->execution_memory_with_exception); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_, + &query_execution->execution_memory); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareIndexQuery(std::move(parsed_query), in_explicit_transaction_, + &query_execution->notifications, interpreter_context_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareAuthQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, + interpreter_context_, &*execution_db_accessor_, + &query_execution->execution_memory_with_exception); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, + interpreter_context_, interpreter_context_->db, + &query_execution->execution_memory_with_exception); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareConstraintQuery(std::move(parsed_query), in_explicit_transaction_, + &query_execution->notifications, interpreter_context_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = + PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications, + interpreter_context_, &*execution_db_accessor_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_, + &*execution_db_accessor_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareFreeMemoryQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = + PrepareTriggerQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications, + interpreter_context_, &*execution_db_accessor_, params, username); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = + PrepareStreamQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications, + interpreter_context_, &*execution_db_accessor_, params, username); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = + PrepareIsolationLevelQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_, this); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = + PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, &*execution_db_accessor_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_); + } else { + LOG_FATAL("Should not get here -- unknown query type!"); + } + + query_execution->summary["planning_time"] = planning_timer.Elapsed().count(); + query_execution->prepared_query.emplace(std::move(prepared_query)); + + const auto rw_type = query_execution->prepared_query->rw_type; + query_execution->summary["type"] = plan::ReadWriteTypeChecker::TypeToString(rw_type); + + UpdateTypeCount(rw_type); + + if (const auto query_type = query_execution->prepared_query->rw_type; + interpreter_context_->db->GetReplicationRole() == storage::v3::ReplicationRole::REPLICA && + (query_type == RWType::W || query_type == RWType::RW)) { + query_execution = nullptr; + throw QueryException("Write query forbidden on the replica!"); + } + + return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid}; + } catch (const utils::BasicException &) { + EventCounter::IncrementCounter(EventCounter::FailedQuery); + AbortCommand(&query_execution); + throw; + } +} + +void Interpreter::Abort() { + expect_rollback_ = false; + in_explicit_transaction_ = false; + if (!db_accessor_) return; + db_accessor_->Abort(); + execution_db_accessor_.reset(); + db_accessor_.reset(); + trigger_context_collector_.reset(); +} + +namespace { +void RunTriggersIndividually(const utils::SkipList &triggers, InterpreterContext *interpreter_context, + TriggerContext trigger_context) { + // Run the triggers + for (const auto &trigger : triggers.access()) { + utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize}; + + // create a new transaction for each trigger + auto storage_acc = interpreter_context->db->Access(); + DbAccessor db_accessor{&storage_acc}; + + trigger_context.AdaptForAccessor(&db_accessor); + try { + trigger.Execute(&db_accessor, &execution_memory, interpreter_context->config.execution_timeout_sec, + &interpreter_context->is_shutting_down, trigger_context, interpreter_context->auth_checker); + } catch (const utils::BasicException &exception) { + spdlog::warn("Trigger '{}' failed with exception:\n{}", trigger.Name(), exception.what()); + db_accessor.Abort(); + continue; + } + + auto maybe_constraint_violation = db_accessor.Commit(); + if (maybe_constraint_violation.HasError()) { + const auto &constraint_violation = maybe_constraint_violation.GetError(); + switch (constraint_violation.type) { + case storage::v3::ConstraintViolation::Type::EXISTENCE: { + const auto &label_name = db_accessor.LabelToName(constraint_violation.label); + MG_ASSERT(constraint_violation.properties.size() == 1U); + const auto &property_name = db_accessor.PropertyToName(*constraint_violation.properties.begin()); + spdlog::warn("Trigger '{}' failed to commit due to existence constraint violation on :{}({})", trigger.Name(), + label_name, property_name); + break; + } + case storage::v3::ConstraintViolation::Type::UNIQUE: { + const auto &label_name = db_accessor.LabelToName(constraint_violation.label); + std::stringstream property_names_stream; + utils::PrintIterable(property_names_stream, constraint_violation.properties, ", ", + [&](auto &stream, const auto &prop) { stream << db_accessor.PropertyToName(prop); }); + spdlog::warn("Trigger '{}' failed to commit due to unique constraint violation on :{}({})", trigger.Name(), + label_name, property_names_stream.str()); + break; + } + } + } + } +} +} // namespace + +void Interpreter::Commit() { + // It's possible that some queries did not finish because the user did + // not pull all of the results from the query. + // For now, we will not check if there are some unfinished queries. + // We should document clearly that all results should be pulled to complete + // a query. + if (!db_accessor_) return; + + std::optional trigger_context = std::nullopt; + if (trigger_context_collector_) { + trigger_context.emplace(std::move(*trigger_context_collector_).TransformToTriggerContext()); + trigger_context_collector_.reset(); + } + + if (trigger_context) { + // Run the triggers + for (const auto &trigger : interpreter_context_->trigger_store.BeforeCommitTriggers().access()) { + utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize}; + AdvanceCommand(); + try { + trigger.Execute(&*execution_db_accessor_, &execution_memory, interpreter_context_->config.execution_timeout_sec, + &interpreter_context_->is_shutting_down, *trigger_context, interpreter_context_->auth_checker); + } catch (const utils::BasicException &e) { + throw utils::BasicException( + fmt::format("Trigger '{}' caused the transaction to fail.\nException: {}", trigger.Name(), e.what())); + } + } + SPDLOG_DEBUG("Finished executing before commit triggers"); + } + + const auto reset_necessary_members = [this]() { + execution_db_accessor_.reset(); + db_accessor_.reset(); + trigger_context_collector_.reset(); + }; + + auto maybe_constraint_violation = db_accessor_->Commit(); + if (maybe_constraint_violation.HasError()) { + const auto &constraint_violation = maybe_constraint_violation.GetError(); + switch (constraint_violation.type) { + case storage::v3::ConstraintViolation::Type::EXISTENCE: { + auto label_name = execution_db_accessor_->LabelToName(constraint_violation.label); + MG_ASSERT(constraint_violation.properties.size() == 1U); + auto property_name = execution_db_accessor_->PropertyToName(*constraint_violation.properties.begin()); + reset_necessary_members(); + throw QueryException("Unable to commit due to existence constraint violation on :{}({})", label_name, + property_name); + break; + } + case storage::v3::ConstraintViolation::Type::UNIQUE: { + auto label_name = execution_db_accessor_->LabelToName(constraint_violation.label); + std::stringstream property_names_stream; + utils::PrintIterable( + property_names_stream, constraint_violation.properties, ", ", + [this](auto &stream, const auto &prop) { stream << execution_db_accessor_->PropertyToName(prop); }); + reset_necessary_members(); + throw QueryException("Unable to commit due to unique constraint violation on :{}({})", label_name, + property_names_stream.str()); + break; + } + } + } + + // The ordered execution of after commit triggers is heavily depending on the exclusiveness of db_accessor_->Commit(): + // only one of the transactions can be commiting at the same time, so when the commit is finished, that transaction + // probably will schedule its after commit triggers, because the other transactions that want to commit are still + // waiting for commiting or one of them just started commiting its changes. + // This means the ordered execution of after commit triggers are not guaranteed. + if (trigger_context && interpreter_context_->trigger_store.AfterCommitTriggers().size() > 0) { + interpreter_context_->after_commit_trigger_pool.AddTask( + [trigger_context = std::move(*trigger_context), interpreter_context = this->interpreter_context_, + user_transaction = std::shared_ptr(std::move(db_accessor_))]() mutable { + RunTriggersIndividually(interpreter_context->trigger_store.AfterCommitTriggers(), interpreter_context, + std::move(trigger_context)); + user_transaction->FinalizeTransaction(); + SPDLOG_DEBUG("Finished executing after commit triggers"); // NOLINT(bugprone-lambda-function-name) + }); + } + + reset_necessary_members(); + + SPDLOG_DEBUG("Finished committing the transaction"); +} + +void Interpreter::AdvanceCommand() { + if (!db_accessor_) return; + db_accessor_->AdvanceCommand(); +} + +void Interpreter::AbortCommand(std::unique_ptr *query_execution) { + if (query_execution) { + query_execution->reset(nullptr); + } + if (in_explicit_transaction_) { + expect_rollback_ = true; + } else { + Abort(); + } +} + +std::optional Interpreter::GetIsolationLevelOverride() { + if (next_transaction_isolation_level) { + const auto isolation_level = *next_transaction_isolation_level; + next_transaction_isolation_level.reset(); + return isolation_level; + } + + return interpreter_isolation_level; +} + +void Interpreter::SetNextTransactionIsolationLevel(const storage::v3::IsolationLevel isolation_level) { + next_transaction_isolation_level.emplace(isolation_level); +} + +void Interpreter::SetSessionIsolationLevel(const storage::v3::IsolationLevel isolation_level) { + interpreter_isolation_level.emplace(isolation_level); +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/interpreter.hpp b/src/query/v2/interpreter.hpp new file mode 100644 index 000000000..b1f89f22b --- /dev/null +++ b/src/query/v2/interpreter.hpp @@ -0,0 +1,436 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "query/v2/auth_checker.hpp" +#include "query/v2/config.hpp" +#include "query/v2/context.hpp" +#include "query/v2/cypher_query_interpreter.hpp" +#include "query/v2/db_accessor.hpp" +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/ast/cypher_main_visitor.hpp" +#include "query/v2/frontend/stripped.hpp" +#include "query/v2/interpret/frame.hpp" +#include "query/v2/metadata.hpp" +#include "query/v2/plan/operator.hpp" +#include "query/v2/plan/read_write_type_checker.hpp" +#include "query/v2/stream.hpp" +#include "query/v2/stream/streams.hpp" +#include "query/v2/trigger.hpp" +#include "query/v2/typed_value.hpp" +#include "storage/v3/isolation_level.hpp" +#include "utils/event_counter.hpp" +#include "utils/logging.hpp" +#include "utils/memory.hpp" +#include "utils/settings.hpp" +#include "utils/skip_list.hpp" +#include "utils/spin_lock.hpp" +#include "utils/thread_pool.hpp" +#include "utils/timer.hpp" +#include "utils/tsc.hpp" + +namespace EventCounter { +extern const Event FailedQuery; +} // namespace EventCounter + +namespace memgraph::query::v2 { + +inline constexpr size_t kExecutionMemoryBlockSize = 1UL * 1024UL * 1024UL; + +class AuthQueryHandler { + public: + AuthQueryHandler() = default; + virtual ~AuthQueryHandler() = default; + + AuthQueryHandler(const AuthQueryHandler &) = delete; + AuthQueryHandler(AuthQueryHandler &&) = delete; + AuthQueryHandler &operator=(const AuthQueryHandler &) = delete; + AuthQueryHandler &operator=(AuthQueryHandler &&) = delete; + + /// Return false if the user already exists. + /// @throw QueryRuntimeException if an error ocurred. + virtual bool CreateUser(const std::string &username, const std::optional &password) = 0; + + /// Return false if the user does not exist. + /// @throw QueryRuntimeException if an error ocurred. + virtual bool DropUser(const std::string &username) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void SetPassword(const std::string &username, const std::optional &password) = 0; + + /// Return false if the role already exists. + /// @throw QueryRuntimeException if an error ocurred. + virtual bool CreateRole(const std::string &rolename) = 0; + + /// Return false if the role does not exist. + /// @throw QueryRuntimeException if an error ocurred. + virtual bool DropRole(const std::string &rolename) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual std::vector GetUsernames() = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual std::vector GetRolenames() = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual std::optional GetRolenameForUser(const std::string &username) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual std::vector GetUsernamesForRole(const std::string &rolename) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void SetRole(const std::string &username, const std::string &rolename) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void ClearRole(const std::string &username) = 0; + + virtual std::vector> GetPrivileges(const std::string &user_or_role) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void GrantPrivilege(const std::string &user_or_role, const std::vector &privileges) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void DenyPrivilege(const std::string &user_or_role, const std::vector &privileges) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void RevokePrivilege(const std::string &user_or_role, + const std::vector &privileges) = 0; +}; + +enum class QueryHandlerResult { COMMIT, ABORT, NOTHING }; + +class ReplicationQueryHandler { + public: + ReplicationQueryHandler() = default; + virtual ~ReplicationQueryHandler() = default; + + ReplicationQueryHandler(const ReplicationQueryHandler &) = default; + ReplicationQueryHandler &operator=(const ReplicationQueryHandler &) = default; + + ReplicationQueryHandler(ReplicationQueryHandler &&) = default; + ReplicationQueryHandler &operator=(ReplicationQueryHandler &&) = default; + + struct Replica { + std::string name; + std::string socket_address; + ReplicationQuery::SyncMode sync_mode; + std::optional timeout; + ReplicationQuery::ReplicaState state; + }; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional port) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual ReplicationQuery::ReplicationRole ShowReplicationRole() const = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void RegisterReplica(const std::string &name, const std::string &socket_address, + const ReplicationQuery::SyncMode sync_mode, const std::optional timeout, + const std::chrono::seconds replica_check_frequency) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual void DropReplica(const std::string &replica_name) = 0; + + /// @throw QueryRuntimeException if an error ocurred. + virtual std::vector ShowReplicas() const = 0; +}; + +/** + * A container for data related to the preparation of a query. + */ +struct PreparedQuery { + std::vector header; + std::vector privileges; + std::function(AnyStream *stream, std::optional n)> query_handler; + plan::ReadWriteTypeChecker::RWType rw_type; +}; + +/** + * Holds data shared between multiple `Interpreter` instances (which might be + * running concurrently). + * + * Users should initialize the context but should not modify it after it has + * been passed to an `Interpreter` instance. + */ +struct InterpreterContext { + explicit InterpreterContext(storage::v3::Storage *db, InterpreterConfig config, + const std::filesystem::path &data_directory); + + storage::v3::Storage *db; + + // ANTLR has singleton instance that is shared between threads. It is + // protected by locks inside of ANTLR. Unfortunately, they are not protected + // in a very good way. Once we have ANTLR version without race conditions we + // can remove this lock. This will probably never happen since ANTLR + // developers introduce more bugs in each version. Fortunately, we have + // cache so this lock probably won't impact performance much... + utils::SpinLock antlr_lock; + std::optional tsc_frequency{utils::GetTSCFrequency()}; + std::atomic is_shutting_down{false}; + + AuthQueryHandler *auth{nullptr}; + AuthChecker *auth_checker{nullptr}; + + utils::SkipList ast_cache; + utils::SkipList plan_cache; + + TriggerStore trigger_store; + utils::ThreadPool after_commit_trigger_pool{1}; + + const InterpreterConfig config; + + query::v2::stream::Streams streams; +}; + +/// Function that is used to tell all active interpreters that they should stop +/// their ongoing execution. +inline void Shutdown(InterpreterContext *context) { context->is_shutting_down.store(true, std::memory_order_release); } + +class Interpreter final { + public: + explicit Interpreter(InterpreterContext *interpreter_context); + Interpreter(const Interpreter &) = delete; + Interpreter &operator=(const Interpreter &) = delete; + Interpreter(Interpreter &&) = delete; + Interpreter &operator=(Interpreter &&) = delete; + ~Interpreter() { Abort(); } + + struct PrepareResult { + std::vector headers; + std::vector privileges; + std::optional qid; + }; + + /** + * Prepare a query for execution. + * + * Preparing a query means to preprocess the query and save it for + * future calls of `Pull`. + * + * @throw query::v2::QueryException + */ + PrepareResult Prepare(const std::string &query, const std::map ¶ms, + const std::string *username); + + /** + * Execute the last prepared query and stream *all* of the results into the + * given stream. + * + * It is not possible to prepare a query once and execute it multiple times, + * i.e. `Prepare` has to be called before *every* call to `PullAll`. + * + * TStream should be a type implementing the `Stream` concept, i.e. it should + * contain the member function `void Result(const std::vector &)`. + * The provided vector argument is valid only for the duration of the call to + * `Result`. The stream should make an explicit copy if it wants to use it + * further. + * + * @throw utils::BasicException + * @throw query::v2::QueryException + */ + template + std::map PullAll(TStream *result_stream) { + return Pull(result_stream); + } + + /** + * Execute a prepared query and stream result into the given stream. + * + * TStream should be a type implementing the `Stream` concept, i.e. it should + * contain the member function `void Result(const std::vector &)`. + * The provided vector argument is valid only for the duration of the call to + * `Result`. The stream should make an explicit copy if it wants to use it + * further. + * + * @param n If set, amount of rows to be pulled from result, + * otherwise all the rows are pulled. + * @param qid If set, id of the query from which the result should be pulled, + * otherwise the last query should be used. + * + * @throw utils::BasicException + * @throw query::v2::QueryException + */ + template + std::map Pull(TStream *result_stream, std::optional n = {}, + std::optional qid = {}); + + void BeginTransaction(); + + void CommitTransaction(); + + void RollbackTransaction(); + + void SetNextTransactionIsolationLevel(storage::v3::IsolationLevel isolation_level); + void SetSessionIsolationLevel(storage::v3::IsolationLevel isolation_level); + + /** + * Abort the current multicommand transaction. + */ + void Abort(); + + private: + struct QueryExecution { + std::optional prepared_query; + utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize}; + utils::ResourceWithOutOfMemoryException execution_memory_with_exception{&execution_memory}; + + std::map summary; + std::vector notifications; + + explicit QueryExecution() = default; + QueryExecution(const QueryExecution &) = delete; + QueryExecution(QueryExecution &&) = default; + QueryExecution &operator=(const QueryExecution &) = delete; + QueryExecution &operator=(QueryExecution &&) = default; + + ~QueryExecution() { + // We should always release the execution memory AFTER we + // destroy the prepared query which is using that instance + // of execution memory. + prepared_query.reset(); + execution_memory.Release(); + } + }; + + // Interpreter supports multiple prepared queries at the same time. + // The client can reference a specific query for pull using an arbitrary qid + // which is in our case the index of the query in the vector. + // To simplify the handling of the qid we avoid modifying the vector if it + // affects the position of the currently running queries in any way. + // For example, we cannot delete the prepared query from the vector because + // every prepared query after the deleted one will be moved by one place + // making their qid not equal to the their index inside the vector. + // To avoid this, we use unique_ptr with which we manualy control construction + // and deletion of a single query execution, i.e. when a query finishes, + // we reset the corresponding unique_ptr. + std::vector> query_executions_; + + InterpreterContext *interpreter_context_; + + // This cannot be std::optional because we need to move this accessor later on into a lambda capture + // which is assigned to std::function. std::function requires every object to be copyable, so we + // move this unique_ptr into a shrared_ptr. + std::unique_ptr db_accessor_; + std::optional execution_db_accessor_; + std::optional trigger_context_collector_; + bool in_explicit_transaction_{false}; + bool expect_rollback_{false}; + + std::optional interpreter_isolation_level; + std::optional next_transaction_isolation_level; + + PreparedQuery PrepareTransactionQuery(std::string_view query_upper); + void Commit(); + void AdvanceCommand(); + void AbortCommand(std::unique_ptr *query_execution); + std::optional GetIsolationLevelOverride(); + + size_t ActiveQueryExecutions() { + return std::count_if(query_executions_.begin(), query_executions_.end(), + [](const auto &execution) { return execution && execution->prepared_query; }); + } +}; + +template +std::map Interpreter::Pull(TStream *result_stream, std::optional n, + std::optional qid) { + MG_ASSERT(in_explicit_transaction_ || !qid, "qid can be only used in explicit transaction!"); + const int qid_value = qid ? *qid : static_cast(query_executions_.size() - 1); + + if (qid_value < 0 || qid_value >= query_executions_.size()) { + throw InvalidArgumentsException("qid", "Query with specified ID does not exist!"); + } + + if (n && n < 0) { + throw InvalidArgumentsException("n", "Cannot fetch negative number of results!"); + } + + auto &query_execution = query_executions_[qid_value]; + + MG_ASSERT(query_execution && query_execution->prepared_query, "Query already finished executing!"); + + // Each prepared query has its own summary so we need to somehow preserve + // it after it finishes executing because it gets destroyed alongside + // the prepared query and its execution memory. + std::optional> maybe_summary; + try { + // Wrap the (statically polymorphic) stream type into a common type which + // the handler knows. + AnyStream stream{result_stream, &query_execution->execution_memory}; + const auto maybe_res = query_execution->prepared_query->query_handler(&stream, n); + // Stream is using execution memory of the query_execution which + // can be deleted after its execution so the stream should be cleared + // first. + stream.~AnyStream(); + + // If the query finished executing, we have received a value which tells + // us what to do after. + if (maybe_res) { + // Save its summary + maybe_summary.emplace(std::move(query_execution->summary)); + if (!query_execution->notifications.empty()) { + std::vector notifications; + notifications.reserve(query_execution->notifications.size()); + for (const auto ¬ification : query_execution->notifications) { + notifications.emplace_back(notification.ConvertToMap()); + } + maybe_summary->insert_or_assign("notifications", std::move(notifications)); + } + if (!in_explicit_transaction_) { + switch (*maybe_res) { + case QueryHandlerResult::COMMIT: + Commit(); + break; + case QueryHandlerResult::ABORT: + Abort(); + break; + case QueryHandlerResult::NOTHING: + // The only cases in which we have nothing to do are those where + // we're either in an explicit transaction or the query is such that + // a transaction wasn't started on a call to `Prepare()`. + MG_ASSERT(in_explicit_transaction_ || !db_accessor_); + break; + } + // As the transaction is done we can clear all the executions + // NOTE: we cannot clear query_execution inside the Abort and Commit + // methods as we will delete summary contained in them which we need + // after our query finished executing. + query_executions_.clear(); + } else { + // We can only clear this execution as some of the queries + // in the transaction can be in unfinished state + query_execution.reset(nullptr); + } + } + } catch (const ExplicitTransactionUsageException &) { + query_execution.reset(nullptr); + throw; + } catch (const utils::BasicException &) { + EventCounter::IncrementCounter(EventCounter::FailedQuery); + AbortCommand(&query_execution); + throw; + } + + if (maybe_summary) { + // return the execution summary + maybe_summary->insert_or_assign("has_more", false); + return std::move(*maybe_summary); + } + + // don't return the execution summary as it's not finished + return {{"has_more", TypedValue(true)}}; +} +} // namespace memgraph::query::v2 diff --git a/src/query/v2/metadata.cpp b/src/query/v2/metadata.cpp new file mode 100644 index 000000000..fe7461e79 --- /dev/null +++ b/src/query/v2/metadata.cpp @@ -0,0 +1,117 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/metadata.hpp" + +#include +#include +#include +#include + +namespace memgraph::query::v2 { + +namespace { +using namespace std::literals; + +constexpr std::string_view GetSeverityLevelString(const SeverityLevel level) { + switch (level) { + case SeverityLevel::INFO: + return "INFO"sv; + case SeverityLevel::WARNING: + return "WARNING"sv; + } +} + +constexpr std::string_view GetCodeString(const NotificationCode code) { + switch (code) { + case NotificationCode::CREATE_CONSTRAINT: + return "CreateConstraint"sv; + case NotificationCode::CREATE_INDEX: + return "CreateIndex"sv; + case NotificationCode::CREATE_STREAM: + return "CreateStream"sv; + case NotificationCode::CHECK_STREAM: + return "CheckStream"sv; + case NotificationCode::CREATE_TRIGGER: + return "CreateTrigger"sv; + case NotificationCode::DROP_CONSTRAINT: + return "DropConstraint"sv; + case NotificationCode::DROP_REPLICA: + return "DropReplica"sv; + case NotificationCode::DROP_INDEX: + return "DropIndex"sv; + case NotificationCode::DROP_STREAM: + return "DropStream"sv; + case NotificationCode::DROP_TRIGGER: + return "DropTrigger"sv; + case NotificationCode::EXISTANT_CONSTRAINT: + return "ConstraintAlreadyExists"sv; + case NotificationCode::EXISTANT_INDEX: + return "IndexAlreadyExists"sv; + case NotificationCode::LOAD_CSV_TIP: + return "LoadCSVTip"sv; + case NotificationCode::NONEXISTANT_INDEX: + return "IndexDoesNotExist"sv; + case NotificationCode::NONEXISTANT_CONSTRAINT: + return "ConstraintDoesNotExist"sv; + case NotificationCode::REGISTER_REPLICA: + return "RegisterReplica"sv; + case NotificationCode::REPLICA_PORT_WARNING: + return "ReplicaPortWarning"sv; + case NotificationCode::SET_REPLICA: + return "SetReplica"sv; + case NotificationCode::START_STREAM: + return "StartStream"sv; + case NotificationCode::START_ALL_STREAMS: + return "StartAllStreams"sv; + case NotificationCode::STOP_STREAM: + return "StopStream"sv; + case NotificationCode::STOP_ALL_STREAMS: + return "StopAllStreams"sv; + } +} +} // namespace + +Notification::Notification(SeverityLevel level) : level{level} {}; + +Notification::Notification(SeverityLevel level, NotificationCode code, std::string title, std::string description) + : level{level}, code{code}, title(std::move(title)), description(std::move(description)){}; + +Notification::Notification(SeverityLevel level, NotificationCode code, std::string title) + : level{level}, code{code}, title(std::move(title)){}; + +std::map Notification::ConvertToMap() const { + return std::map{{"severity", TypedValue(GetSeverityLevelString(level))}, + {"code", TypedValue(GetCodeString(code))}, + {"title", TypedValue(title)}, + {"description", TypedValue(description)}}; +} + +std::string ExecutionStatsKeyToString(const ExecutionStats::Key key) { + switch (key) { + case ExecutionStats::Key::CREATED_NODES: + return std::string("nodes-created"); + case ExecutionStats::Key::DELETED_NODES: + return std::string("nodes-deleted"); + case ExecutionStats::Key::CREATED_EDGES: + return std::string("relationships-created"); + case ExecutionStats::Key::DELETED_EDGES: + return std::string("relationships-deleted"); + case ExecutionStats::Key::CREATED_LABELS: + return std::string("labels-added"); + case ExecutionStats::Key::DELETED_LABELS: + return std::string("labels-removed"); + case ExecutionStats::Key::UPDATED_PROPERTIES: + return std::string("properties-set"); + } +} + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/metadata.hpp b/src/query/v2/metadata.hpp new file mode 100644 index 000000000..ffc621d64 --- /dev/null +++ b/src/query/v2/metadata.hpp @@ -0,0 +1,90 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include +#include +#include + +#include "query/v2/typed_value.hpp" + +namespace memgraph::query::v2 { + +enum class SeverityLevel : uint8_t { INFO, WARNING }; + +enum class NotificationCode : uint8_t { + CREATE_CONSTRAINT, + CREATE_INDEX, + CHECK_STREAM, + CREATE_STREAM, + CREATE_TRIGGER, + DROP_CONSTRAINT, + DROP_INDEX, + DROP_REPLICA, + DROP_STREAM, + DROP_TRIGGER, + EXISTANT_INDEX, + EXISTANT_CONSTRAINT, + LOAD_CSV_TIP, + NONEXISTANT_INDEX, + NONEXISTANT_CONSTRAINT, + REPLICA_PORT_WARNING, + REGISTER_REPLICA, + SET_REPLICA, + START_STREAM, + START_ALL_STREAMS, + STOP_STREAM, + STOP_ALL_STREAMS, +}; + +struct Notification { + SeverityLevel level; + NotificationCode code; + std::string title; + std::string description; + + explicit Notification(SeverityLevel level); + + Notification(SeverityLevel level, NotificationCode code, std::string title, std::string description); + + Notification(SeverityLevel level, NotificationCode code, std::string title); + + std::map ConvertToMap() const; +}; + +struct ExecutionStats { + public: + // All the stats have specific key to be compatible with neo4j + enum class Key : uint8_t { + CREATED_NODES, + DELETED_NODES, + CREATED_EDGES, + DELETED_EDGES, + CREATED_LABELS, + DELETED_LABELS, + UPDATED_PROPERTIES, + }; + + int64_t &operator[](Key key) { return counters[static_cast(key)]; } + + private: + static constexpr auto kExecutionStatsCountersSize = std::underlying_type_t(Key::UPDATED_PROPERTIES) + 1; + + public: + std::array counters{0}; +}; + +std::string ExecutionStatsKeyToString(ExecutionStats::Key key); + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/parameters.hpp b/src/query/v2/parameters.hpp new file mode 100644 index 000000000..1e2f0744f --- /dev/null +++ b/src/query/v2/parameters.hpp @@ -0,0 +1,71 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +#include "storage/v3/property_value.hpp" +#include "utils/logging.hpp" + +/** + * Encapsulates user provided parameters (and stripped literals) + * and provides ways of obtaining them by position. + */ +namespace memgraph::query::v2 { + +struct Parameters { + public: + /** + * Adds a value to the stripped arguments under a token position. + * + * @param position Token position in query of value. + * @param value + */ + void Add(int position, const storage::v3::PropertyValue &value) { storage_.emplace_back(position, value); } + + /** + * Returns the value found for the given token position. + * + * @param position Token position in query of value. + * @return Value for the given token position. + */ + const storage::v3::PropertyValue &AtTokenPosition(int position) const { + auto found = std::find_if(storage_.begin(), storage_.end(), [&](const auto &a) { return a.first == position; }); + MG_ASSERT(found != storage_.end(), "Token position must be present in container"); + return found->second; + } + + /** + * Returns the position-th stripped value. Asserts that this + * container has at least (position + 1) elements. + * + * @param position Which stripped param is sought. + * @return Token position and value for sought param. + */ + const std::pair &At(int position) const { + MG_ASSERT(position < static_cast(storage_.size()), "Invalid position"); + return storage_[position]; + } + + /** Returns the number of arguments in this container */ + auto size() const { return storage_.size(); } + + auto begin() const { return storage_.begin(); } + auto end() const { return storage_.end(); } + + private: + std::vector> storage_; +}; + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/path.hpp b/src/query/v2/path.hpp new file mode 100644 index 000000000..d16e4bba8 --- /dev/null +++ b/src/query/v2/path.hpp @@ -0,0 +1,146 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include + +#include "query/v2/db_accessor.hpp" +#include "utils/logging.hpp" +#include "utils/memory.hpp" +#include "utils/pmr/vector.hpp" + +namespace memgraph::query::v2 { + +/** + * A data structure that holds a graph path. A path consists of at least one + * vertex, followed by zero or more edge + vertex extensions (thus having one + * vertex more then edges). + */ +class Path { + public: + /** Allocator type so that STL containers are aware that we need one */ + using allocator_type = utils::Allocator; + + /** + * Create the path starting with the given vertex. + * Allocations are done using the given MemoryResource. + */ + explicit Path(const VertexAccessor &vertex, utils::MemoryResource *memory = utils::NewDeleteResource()) + : vertices_(memory), edges_(memory) { + Expand(vertex); + } + + /** + * Create the path starting with the given vertex and containing all other + * elements. + * Allocations are done using the default utils::NewDeleteResource(). + */ + template + explicit Path(const VertexAccessor &vertex, const TOthers &...others) + : vertices_(utils::NewDeleteResource()), edges_(utils::NewDeleteResource()) { + Expand(vertex); + Expand(others...); + } + + /** + * Create the path starting with the given vertex and containing all other + * elements. + * Allocations are done using the given MemoryResource. + */ + template + Path(std::allocator_arg_t, utils::MemoryResource *memory, const VertexAccessor &vertex, const TOthers &...others) + : vertices_(memory), edges_(memory) { + Expand(vertex); + Expand(others...); + } + + /** + * Construct a copy of other. + * utils::MemoryResource is obtained by calling + * std::allocator_traits<>:: + * select_on_container_copy_construction(other.GetMemoryResource()). + * Since we use utils::Allocator, which does not propagate, this means that we + * will default to utils::NewDeleteResource(). + */ + Path(const Path &other) + : Path(other, + std::allocator_traits::select_on_container_copy_construction(other.GetMemoryResource()) + .GetMemoryResource()) {} + + /** Construct a copy using the given utils::MemoryResource */ + Path(const Path &other, utils::MemoryResource *memory) + : vertices_(other.vertices_, memory), edges_(other.edges_, memory) {} + + /** + * Construct with the value of other. + * utils::MemoryResource is obtained from other. After the move, other will be + * empty. + */ + Path(Path &&other) noexcept : Path(std::move(other), other.GetMemoryResource()) {} + + /** + * Construct with the value of other, but use the given utils::MemoryResource. + * After the move, other may not be empty if `*memory != + * *other.GetMemoryResource()`, because an element-wise move will be + * performed. + */ + Path(Path &&other, utils::MemoryResource *memory) + : vertices_(std::move(other.vertices_), memory), edges_(std::move(other.edges_), memory) {} + + /** Copy assign other, utils::MemoryResource of `this` is used */ + Path &operator=(const Path &) = default; + + /** Move assign other, utils::MemoryResource of `this` is used. */ + Path &operator=(Path &&) = default; + + ~Path() = default; + + /** Expands the path with the given vertex. */ + void Expand(const VertexAccessor &vertex) { + DMG_ASSERT(vertices_.size() == edges_.size(), "Illegal path construction order"); + vertices_.emplace_back(vertex); + } + + /** Expands the path with the given edge. */ + void Expand(const EdgeAccessor &edge) { + DMG_ASSERT(vertices_.size() - 1 == edges_.size(), "Illegal path construction order"); + edges_.emplace_back(edge); + } + + /** Expands the path with the given elements. */ + template + void Expand(const TFirst &first, const TOthers &...others) { + Expand(first); + Expand(others...); + } + + /** Returns the number of expansions (edges) in this path. */ + auto size() const { return edges_.size(); } + + auto &vertices() { return vertices_; } + auto &edges() { return edges_; } + const auto &vertices() const { return vertices_; } + const auto &edges() const { return edges_; } + + utils::MemoryResource *GetMemoryResource() const { return vertices_.get_allocator().GetMemoryResource(); } + + bool operator==(const Path &other) const { return vertices_ == other.vertices_ && edges_ == other.edges_; } + + private: + // Contains all the vertices in the path. + utils::pmr::vector vertices_; + // Contains all the edges in the path (one less then there are vertices). + utils::pmr::vector edges_; +}; + +} // namespace memgraph::query::v2 diff --git a/src/query/v2/plan/cost_estimator.hpp b/src/query/v2/plan/cost_estimator.hpp new file mode 100644 index 000000000..07a5fde0b --- /dev/null +++ b/src/query/v2/plan/cost_estimator.hpp @@ -0,0 +1,267 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/parameters.hpp" +#include "query/v2/plan/operator.hpp" +#include "query/v2/typed_value.hpp" + +namespace memgraph::query::v2::plan { + +/** + * Query plan execution time cost estimator, for comparing and choosing optimal + * execution plans. + * + * In Cypher the write part of the query always executes in the same + * cardinality. It is not allowed to execute a write operation before all the + * expansion for that query part (WITH splits a query into parts) have executed. + * For that reason cost estimation comes down to cardinality estimation for the + * read parts of the query, and their expansion. We want to compare different + * plans and try to figure out which has the optimal organization of scans, + * expansions and filters. + * + * Note that expansions and filtering can also happen during Merge, which is a + * write operation. We let that get evaluated like all other cardinality + * influencing ops. Also, Merge cardinality modification should be contained (it + * can never reduce it's input cardinality), but since Merge always happens + * after the read part, and can't be reoredered, we can ignore that. + * + * Limiting and accumulating (Aggregate, OrderBy, Accumulate) operations are + * cardinality modifiers that always execute at the end of the query part. Their + * cardinality influence is irrelevant because they execute the same + * for all plans for a single query part, and query part reordering is not + * allowed. + * + * This kind of cost estimation can only be used for comparing logical plans. + * It's aim is to estimate cost(A) to be less then cost(B) in every case where + * actual query execution for plan A is less then that of plan B. It can NOT be + * used to estimate how MUCH execution between A and B will differ. + */ +template +class CostEstimator : public HierarchicalLogicalOperatorVisitor { + public: + struct CostParam { + static constexpr double kScanAll{1.0}; + static constexpr double kScanAllByLabel{1.1}; + static constexpr double MakeScanAllByLabelPropertyValue{1.1}; + static constexpr double MakeScanAllByLabelPropertyRange{1.1}; + static constexpr double MakeScanAllByLabelProperty{1.1}; + static constexpr double kExpand{2.0}; + static constexpr double kExpandVariable{3.0}; + static constexpr double kFilter{1.5}; + static constexpr double kEdgeUniquenessFilter{1.5}; + static constexpr double kUnwind{1.3}; + static constexpr double kForeach{1.0}; + }; + + struct CardParam { + static constexpr double kExpand{3.0}; + static constexpr double kExpandVariable{9.0}; + static constexpr double kFilter{0.25}; + static constexpr double kEdgeUniquenessFilter{0.95}; + }; + + struct MiscParam { + static constexpr double kUnwindNoLiteral{10.0}; + static constexpr double kForeachNoLiteral{10.0}; + }; + + using HierarchicalLogicalOperatorVisitor::PostVisit; + using HierarchicalLogicalOperatorVisitor::PreVisit; + + CostEstimator(TDbAccessor *db_accessor, const Parameters ¶meters) + : db_accessor_(db_accessor), parameters(parameters) {} + + bool PostVisit(ScanAll &) override { + cardinality_ *= db_accessor_->VerticesCount(); + // ScanAll performs some work for every element that is produced + IncrementCost(CostParam::kScanAll); + return true; + } + + bool PostVisit(ScanAllByLabel &scan_all_by_label) override { + cardinality_ *= db_accessor_->VerticesCount(scan_all_by_label.label_); + // ScanAll performs some work for every element that is produced + IncrementCost(CostParam::kScanAllByLabel); + return true; + } + + bool PostVisit(ScanAllByLabelPropertyValue &logical_op) override { + // This cardinality estimation depends on the property value (expression). + // If it's a constant, we can evaluate cardinality exactly, otherwise + // we estimate + auto property_value = ConstPropertyValue(logical_op.expression_); + double factor = 1.0; + if (property_value) + // get the exact influence based on ScanAll(label, property, value) + factor = db_accessor_->VerticesCount(logical_op.label_, logical_op.property_, property_value.value()); + else + // estimate the influence as ScanAll(label, property) * filtering + factor = db_accessor_->VerticesCount(logical_op.label_, logical_op.property_) * CardParam::kFilter; + + cardinality_ *= factor; + + // ScanAll performs some work for every element that is produced + IncrementCost(CostParam::MakeScanAllByLabelPropertyValue); + return true; + } + + bool PostVisit(ScanAllByLabelPropertyRange &logical_op) override { + // this cardinality estimation depends on Bound expressions. + // if they are literals we can evaluate cardinality properly + auto lower = BoundToPropertyValue(logical_op.lower_bound_); + auto upper = BoundToPropertyValue(logical_op.upper_bound_); + + int64_t factor = 1; + if (upper || lower) + // if we have either Bound, use the value index + factor = db_accessor_->VerticesCount(logical_op.label_, logical_op.property_, lower, upper); + else + // no values, but we still have the label + factor = db_accessor_->VerticesCount(logical_op.label_, logical_op.property_); + + // if we failed to take either bound from the op into account, then apply + // the filtering constant to the factor + if ((logical_op.upper_bound_ && !upper) || (logical_op.lower_bound_ && !lower)) factor *= CardParam::kFilter; + + cardinality_ *= factor; + + // ScanAll performs some work for every element that is produced + IncrementCost(CostParam::MakeScanAllByLabelPropertyRange); + return true; + } + + bool PostVisit(ScanAllByLabelProperty &logical_op) override { + const auto factor = db_accessor_->VerticesCount(logical_op.label_, logical_op.property_); + cardinality_ *= factor; + IncrementCost(CostParam::MakeScanAllByLabelProperty); + return true; + } + + // TODO: Cost estimate ScanAllById? + +// For the given op first increments the cardinality and then cost. +#define POST_VISIT_CARD_FIRST(NAME) \ + bool PostVisit(NAME &) override { \ + cardinality_ *= CardParam::k##NAME; \ + IncrementCost(CostParam::k##NAME); \ + return true; \ + } + + POST_VISIT_CARD_FIRST(Expand); + POST_VISIT_CARD_FIRST(ExpandVariable); + +#undef POST_VISIT_CARD_FIRST + +// For the given op first increments the cost and then cardinality. +#define POST_VISIT_COST_FIRST(LOGICAL_OP, PARAM_NAME) \ + bool PostVisit(LOGICAL_OP &) override { \ + IncrementCost(CostParam::PARAM_NAME); \ + cardinality_ *= CardParam::PARAM_NAME; \ + return true; \ + } + + POST_VISIT_COST_FIRST(Filter, kFilter) + POST_VISIT_COST_FIRST(EdgeUniquenessFilter, kEdgeUniquenessFilter); + +#undef POST_VISIT_COST_FIRST + + bool PostVisit(Unwind &unwind) override { + // Unwind cost depends more on the number of lists that get unwound + // much less on the number of outputs + // for that reason first increment cost, then modify cardinality + IncrementCost(CostParam::kUnwind); + + // try to determine how many values will be yielded by Unwind + // if the Unwind expression is a list literal, we can deduce cardinality + // exactly, otherwise we approximate + double unwind_value; + if (auto *literal = utils::Downcast(unwind.input_expression_)) + unwind_value = literal->elements_.size(); + else + unwind_value = MiscParam::kUnwindNoLiteral; + + cardinality_ *= unwind_value; + return true; + } + + bool PostVisit(Foreach &foreach) override { + // Foreach cost depends both on the number elements in the list that get unwound + // as well as the total clauses that get called for each unwounded element. + // First estimate cardinality and then increment the cost. + + double foreach_elements{0}; + if (auto *literal = utils::Downcast(foreach.expression_)) { + foreach_elements = literal->elements_.size(); + } else { + foreach_elements = MiscParam::kForeachNoLiteral; + } + + cardinality_ *= foreach_elements; + IncrementCost(CostParam::kForeach); + return true; + } + + bool Visit(Once &) override { return true; } + + auto cost() const { return cost_; } + auto cardinality() const { return cardinality_; } + + private: + // cost estimation that gets accumulated as the visitor + // tours the logical plan + double cost_{0}; + + // cardinality estimation (how many times an operator gets executed) + // cardinality is a double to make it easier to work with + double cardinality_{1}; + + // accessor used for cardinality estimates in ScanAll and ScanAllByLabel + TDbAccessor *db_accessor_; + const Parameters ¶meters; + + void IncrementCost(double param) { cost_ += param * cardinality_; } + + // converts an optional ScanAll range bound into a property value + // if the bound is present and is a constant expression convertible to + // a property value. otherwise returns nullopt + std::optional> BoundToPropertyValue( + std::optional bound) { + if (bound) { + auto property_value = ConstPropertyValue(bound->value()); + if (property_value) return utils::Bound(*property_value, bound->type()); + } + return std::nullopt; + } + + // If the expression is a constant property value, it is returned. Otherwise, + // return nullopt. + std::optional ConstPropertyValue(const Expression *expression) { + if (auto *literal = utils::Downcast(expression)) { + return literal->value_; + } else if (auto *param_lookup = utils::Downcast(expression)) { + return parameters.AtTokenPosition(param_lookup->token_position_); + } + return std::nullopt; + } +}; + +/** Returns the estimated cost of the given plan. */ +template +double EstimatePlanCost(TDbAccessor *db, const Parameters ¶meters, LogicalOperator &plan) { + CostEstimator estimator(db, parameters); + plan.Accept(estimator); + return estimator.cost(); +} + +} // namespace memgraph::query::v2::plan diff --git a/src/query/v2/plan/operator.cpp b/src/query/v2/plan/operator.cpp new file mode 100644 index 000000000..4dd0bf693 --- /dev/null +++ b/src/query/v2/plan/operator.cpp @@ -0,0 +1,4081 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "query/v2/plan/operator.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "query/v2/context.hpp" +#include "query/v2/db_accessor.hpp" +#include "query/v2/exceptions.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/semantic/symbol_table.hpp" +#include "query/v2/interpret/eval.hpp" +#include "query/v2/path.hpp" +#include "query/v2/plan/scoped_profile.hpp" +#include "query/v2/procedure/cypher_types.hpp" +#include "query/v2/procedure/mg_procedure_impl.hpp" +#include "query/v2/procedure/module.hpp" +#include "storage/v3/property_value.hpp" +#include "utils/algorithm.hpp" +#include "utils/csv_parsing.hpp" +#include "utils/event_counter.hpp" +#include "utils/exceptions.hpp" +#include "utils/fnv.hpp" +#include "utils/likely.hpp" +#include "utils/logging.hpp" +#include "utils/memory.hpp" +#include "utils/pmr/unordered_map.hpp" +#include "utils/pmr/unordered_set.hpp" +#include "utils/pmr/vector.hpp" +#include "utils/readable_size.hpp" +#include "utils/string.hpp" +#include "utils/temporal.hpp" + +// macro for the default implementation of LogicalOperator::Accept +// that accepts the visitor and visits it's input_ operator +#define ACCEPT_WITH_INPUT(class_name) \ + bool class_name::Accept(HierarchicalLogicalOperatorVisitor &visitor) { \ + if (visitor.PreVisit(*this)) { \ + input_->Accept(visitor); \ + } \ + return visitor.PostVisit(*this); \ + } + +#define WITHOUT_SINGLE_INPUT(class_name) \ + bool class_name::HasSingleInput() const { return false; } \ + std::shared_ptr class_name::input() const { \ + LOG_FATAL("Operator " #class_name " has no single input!"); \ + } \ + void class_name::set_input(std::shared_ptr) { \ + LOG_FATAL("Operator " #class_name " has no single input!"); \ + } + +namespace EventCounter { +extern const Event OnceOperator; +extern const Event CreateNodeOperator; +extern const Event CreateExpandOperator; +extern const Event ScanAllOperator; +extern const Event ScanAllByLabelOperator; +extern const Event ScanAllByLabelPropertyRangeOperator; +extern const Event ScanAllByLabelPropertyValueOperator; +extern const Event ScanAllByLabelPropertyOperator; +extern const Event ScanAllByIdOperator; +extern const Event ExpandOperator; +extern const Event ExpandVariableOperator; +extern const Event ConstructNamedPathOperator; +extern const Event FilterOperator; +extern const Event ProduceOperator; +extern const Event DeleteOperator; +extern const Event SetPropertyOperator; +extern const Event SetPropertiesOperator; +extern const Event SetLabelsOperator; +extern const Event RemovePropertyOperator; +extern const Event RemoveLabelsOperator; +extern const Event EdgeUniquenessFilterOperator; +extern const Event AccumulateOperator; +extern const Event AggregateOperator; +extern const Event SkipOperator; +extern const Event LimitOperator; +extern const Event OrderByOperator; +extern const Event MergeOperator; +extern const Event OptionalOperator; +extern const Event UnwindOperator; +extern const Event DistinctOperator; +extern const Event UnionOperator; +extern const Event CartesianOperator; +extern const Event CallProcedureOperator; +extern const Event ForeachOperator; +} // namespace EventCounter + +namespace memgraph::query::v2::plan { + +namespace { + +// Custom equality function for a vector of typed values. +// Used in unordered_maps in Aggregate and Distinct operators. +struct TypedValueVectorEqual { + template + bool operator()(const std::vector &left, + const std::vector &right) const { + MG_ASSERT(left.size() == right.size(), + "TypedValueVector comparison should only be done over vectors " + "of the same size"); + return std::equal(left.begin(), left.end(), right.begin(), TypedValue::BoolEqual{}); + } +}; + +// Returns boolean result of evaluating filter expression. Null is treated as +// false. Other non boolean values raise a QueryRuntimeException. +bool EvaluateFilter(ExpressionEvaluator &evaluator, Expression *filter) { + TypedValue result = filter->Accept(evaluator); + // Null is treated like false. + if (result.IsNull()) return false; + if (result.type() != TypedValue::Type::Bool) + throw QueryRuntimeException("Filter expression must evaluate to bool or null, got {}.", result.type()); + return result.ValueBool(); +} + +template +uint64_t ComputeProfilingKey(const T *obj) { + static_assert(sizeof(T *) == sizeof(uint64_t)); + return reinterpret_cast(obj); +} + +} // namespace + +#define SCOPED_PROFILE_OP(name) ScopedProfile profile{ComputeProfilingKey(this), name, &context}; + +bool Once::OnceCursor::Pull(Frame &, ExecutionContext &context) { + SCOPED_PROFILE_OP("Once"); + + if (!did_pull_) { + did_pull_ = true; + return true; + } + return false; +} + +UniqueCursorPtr Once::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::OnceOperator); + + return MakeUniqueCursorPtr(mem); +} + +WITHOUT_SINGLE_INPUT(Once); + +void Once::OnceCursor::Shutdown() {} + +void Once::OnceCursor::Reset() { did_pull_ = false; } + +CreateNode::CreateNode(const std::shared_ptr &input, const NodeCreationInfo &node_info) + : input_(input ? input : std::make_shared()), node_info_(node_info) {} + +// Creates a vertex on this GraphDb. Returns a reference to vertex placed on the +// frame. +VertexAccessor &CreateLocalVertex(const NodeCreationInfo &node_info, Frame *frame, ExecutionContext &context) { + auto &dba = *context.db_accessor; + auto new_node = dba.InsertVertex(); + context.execution_stats[ExecutionStats::Key::CREATED_NODES] += 1; + for (auto label : node_info.labels) { + auto maybe_error = new_node.AddLabel(label); + if (maybe_error.HasError()) { + switch (maybe_error.GetError()) { + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to set a label on a deleted node."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when setting a label."); + } + } + context.execution_stats[ExecutionStats::Key::CREATED_LABELS] += 1; + } + // Evaluator should use the latest accessors, as modified in this query, when + // setting properties on new nodes. + ExpressionEvaluator evaluator(frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::NEW); + // TODO: PropsSetChecked allocates a PropertyValue, make it use context.memory + // when we update PropertyValue with custom allocator. + if (const auto *node_info_properties = std::get_if(&node_info.properties)) { + for (const auto &[key, value_expression] : *node_info_properties) { + PropsSetChecked(&new_node, key, value_expression->Accept(evaluator)); + } + } else { + auto property_map = evaluator.Visit(*std::get(node_info.properties)); + for (const auto &[key, value] : property_map.ValueMap()) { + auto property_id = dba.NameToProperty(key); + PropsSetChecked(&new_node, property_id, value); + } + } + + (*frame)[node_info.symbol] = new_node; + return (*frame)[node_info.symbol].ValueVertex(); +} + +ACCEPT_WITH_INPUT(CreateNode) + +UniqueCursorPtr CreateNode::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::CreateNodeOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector CreateNode::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.emplace_back(node_info_.symbol); + return symbols; +} + +CreateNode::CreateNodeCursor::CreateNodeCursor(const CreateNode &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +bool CreateNode::CreateNodeCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("CreateNode"); + + if (input_cursor_->Pull(frame, context)) { + auto created_vertex = CreateLocalVertex(self_.node_info_, &frame, context); + if (context.trigger_context_collector) { + context.trigger_context_collector->RegisterCreatedObject(created_vertex); + } + return true; + } + + return false; +} + +void CreateNode::CreateNodeCursor::Shutdown() { input_cursor_->Shutdown(); } + +void CreateNode::CreateNodeCursor::Reset() { input_cursor_->Reset(); } + +CreateExpand::CreateExpand(const NodeCreationInfo &node_info, const EdgeCreationInfo &edge_info, + const std::shared_ptr &input, Symbol input_symbol, bool existing_node) + : node_info_(node_info), + edge_info_(edge_info), + input_(input ? input : std::make_shared()), + input_symbol_(input_symbol), + existing_node_(existing_node) {} + +ACCEPT_WITH_INPUT(CreateExpand) + +UniqueCursorPtr CreateExpand::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::CreateNodeOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector CreateExpand::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.emplace_back(node_info_.symbol); + symbols.emplace_back(edge_info_.symbol); + return symbols; +} + +CreateExpand::CreateExpandCursor::CreateExpandCursor(const CreateExpand &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +namespace { + +EdgeAccessor CreateEdge(const EdgeCreationInfo &edge_info, DbAccessor *dba, VertexAccessor *from, VertexAccessor *to, + Frame *frame, ExpressionEvaluator *evaluator) { + auto maybe_edge = dba->InsertEdge(from, to, edge_info.edge_type); + if (maybe_edge.HasValue()) { + auto &edge = *maybe_edge; + if (const auto *properties = std::get_if(&edge_info.properties)) { + for (const auto &[key, value_expression] : *properties) { + PropsSetChecked(&edge, key, value_expression->Accept(*evaluator)); + } + } else { + auto property_map = evaluator->Visit(*std::get(edge_info.properties)); + for (const auto &[key, value] : property_map.ValueMap()) { + auto property_id = dba->NameToProperty(key); + PropsSetChecked(&edge, property_id, value); + } + } + + (*frame)[edge_info.symbol] = edge; + } else { + switch (maybe_edge.GetError()) { + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to create an edge on a deleted node."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when creating an edge."); + } + } + + return *maybe_edge; +} + +} // namespace + +bool CreateExpand::CreateExpandCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("CreateExpand"); + + if (!input_cursor_->Pull(frame, context)) return false; + + // get the origin vertex + TypedValue &vertex_value = frame[self_.input_symbol_]; + ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex); + auto &v1 = vertex_value.ValueVertex(); + + // Similarly to CreateNode, newly created edges and nodes should use the + // storage::v3::View::NEW. + // E.g. we pickup new properties: `CREATE (n {p: 42}) -[:r {ep: n.p}]-> ()` + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::NEW); + + // get the destination vertex (possibly an existing node) + auto &v2 = OtherVertex(frame, context); + + // create an edge between the two nodes + auto *dba = context.db_accessor; + + auto created_edge = [&] { + switch (self_.edge_info_.direction) { + case EdgeAtom::Direction::IN: + return CreateEdge(self_.edge_info_, dba, &v2, &v1, &frame, &evaluator); + case EdgeAtom::Direction::OUT: + // in the case of an undirected CreateExpand we choose an arbitrary + // direction. this is used in the MERGE clause + // it is not allowed in the CREATE clause, and the semantic + // checker needs to ensure it doesn't reach this point + case EdgeAtom::Direction::BOTH: + return CreateEdge(self_.edge_info_, dba, &v1, &v2, &frame, &evaluator); + } + }(); + + context.execution_stats[ExecutionStats::Key::CREATED_EDGES] += 1; + if (context.trigger_context_collector) { + context.trigger_context_collector->RegisterCreatedObject(created_edge); + } + + return true; +} + +void CreateExpand::CreateExpandCursor::Shutdown() { input_cursor_->Shutdown(); } + +void CreateExpand::CreateExpandCursor::Reset() { input_cursor_->Reset(); } + +VertexAccessor &CreateExpand::CreateExpandCursor::OtherVertex(Frame &frame, ExecutionContext &context) { + if (self_.existing_node_) { + TypedValue &dest_node_value = frame[self_.node_info_.symbol]; + ExpectType(self_.node_info_.symbol, dest_node_value, TypedValue::Type::Vertex); + return dest_node_value.ValueVertex(); + } else { + auto &created_vertex = CreateLocalVertex(self_.node_info_, &frame, context); + if (context.trigger_context_collector) { + context.trigger_context_collector->RegisterCreatedObject(created_vertex); + } + return created_vertex; + } +} + +template +class ScanAllCursor : public Cursor { + public: + explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices, + const char *op_name) + : output_symbol_(output_symbol), + input_cursor_(std::move(input_cursor)), + get_vertices_(std::move(get_vertices)), + op_name_(op_name) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP(op_name_); + + if (MustAbort(context)) throw HintedAbortError(); + + while (!vertices_ || vertices_it_.value() == vertices_.value().end()) { + if (!input_cursor_->Pull(frame, context)) return false; + // We need a getter function, because in case of exhausting a lazy + // iterable, we cannot simply reset it by calling begin(). + auto next_vertices = get_vertices_(frame, context); + if (!next_vertices) continue; + // Since vertices iterator isn't nothrow_move_assignable, we have to use + // the roundabout assignment + emplace, instead of simple: + // vertices _ = get_vertices_(frame, context); + vertices_.emplace(std::move(next_vertices.value())); + vertices_it_.emplace(vertices_.value().begin()); + } + + frame[output_symbol_] = *vertices_it_.value(); + ++vertices_it_.value(); + return true; + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + vertices_ = std::nullopt; + vertices_it_ = std::nullopt; + } + + private: + const Symbol output_symbol_; + const UniqueCursorPtr input_cursor_; + TVerticesFun get_vertices_; + std::optional::type::value_type> vertices_; + std::optional vertices_it_; + const char *op_name_; +}; + +ScanAll::ScanAll(const std::shared_ptr &input, Symbol output_symbol, storage::v3::View view) + : input_(input ? input : std::make_shared()), output_symbol_(output_symbol), view_(view) {} + +ACCEPT_WITH_INPUT(ScanAll) + +UniqueCursorPtr ScanAll::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ScanAllOperator); + + auto vertices = [this](Frame &, ExecutionContext &context) { + auto *db = context.db_accessor; + return std::make_optional(db->Vertices(view_)); + }; + return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), + std::move(vertices), "ScanAll"); +} + +std::vector ScanAll::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.emplace_back(output_symbol_); + return symbols; +} + +ScanAllByLabel::ScanAllByLabel(const std::shared_ptr &input, Symbol output_symbol, + storage::v3::LabelId label, storage::v3::View view) + : ScanAll(input, output_symbol, view), label_(label) {} + +ACCEPT_WITH_INPUT(ScanAllByLabel) + +UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ScanAllByLabelOperator); + + auto vertices = [this](Frame &, ExecutionContext &context) { + auto *db = context.db_accessor; + return std::make_optional(db->Vertices(view_, label_)); + }; + return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), + std::move(vertices), "ScanAllByLabel"); +} + +// TODO(buda): Implement ScanAllByLabelProperty operator to iterate over +// vertices that have the label and some value for the given property. + +ScanAllByLabelPropertyRange::ScanAllByLabelPropertyRange(const std::shared_ptr &input, + Symbol output_symbol, storage::v3::LabelId label, + storage::v3::PropertyId property, + const std::string &property_name, + std::optional lower_bound, + std::optional upper_bound, storage::v3::View view) + : ScanAll(input, output_symbol, view), + label_(label), + property_(property), + property_name_(property_name), + lower_bound_(lower_bound), + upper_bound_(upper_bound) { + MG_ASSERT(lower_bound_ || upper_bound_, "Only one bound can be left out"); +} + +ACCEPT_WITH_INPUT(ScanAllByLabelPropertyRange) + +UniqueCursorPtr ScanAllByLabelPropertyRange::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ScanAllByLabelPropertyRangeOperator); + + auto vertices = [this](Frame &frame, ExecutionContext &context) + -> std::optionalVertices(view_, label_, property_, std::nullopt, std::nullopt))> { + auto *db = context.db_accessor; + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, view_); + auto convert = [&evaluator](const auto &bound) -> std::optional> { + if (!bound) return std::nullopt; + const auto &value = bound->value()->Accept(evaluator); + try { + const auto &property_value = storage::v3::PropertyValue(value); + switch (property_value.type()) { + case storage::v3::PropertyValue::Type::Bool: + case storage::v3::PropertyValue::Type::List: + case storage::v3::PropertyValue::Type::Map: + // Prevent indexed lookup with something that would fail if we did + // the original filter with `operator<`. Note, for some reason, + // Cypher does not support comparing boolean values. + throw QueryRuntimeException("Invalid type {} for '<'.", value.type()); + case storage::v3::PropertyValue::Type::Null: + case storage::v3::PropertyValue::Type::Int: + case storage::v3::PropertyValue::Type::Double: + case storage::v3::PropertyValue::Type::String: + case storage::v3::PropertyValue::Type::TemporalData: + // These are all fine, there's also Point, Date and Time data types + // which were added to Cypher, but we don't have support for those + // yet. + return std::make_optional(utils::Bound(property_value, bound->type())); + } + } catch (const TypedValueException &) { + throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type()); + } + }; + auto maybe_lower = convert(lower_bound_); + auto maybe_upper = convert(upper_bound_); + // If any bound is null, then the comparison would result in nulls. This + // is treated as not satisfying the filter, so return no vertices. + if (maybe_lower && maybe_lower->value().IsNull()) return std::nullopt; + if (maybe_upper && maybe_upper->value().IsNull()) return std::nullopt; + return std::make_optional(db->Vertices(view_, label_, property_, maybe_lower, maybe_upper)); + }; + return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), + std::move(vertices), "ScanAllByLabelPropertyRange"); +} + +ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr &input, + Symbol output_symbol, storage::v3::LabelId label, + storage::v3::PropertyId property, + const std::string &property_name, Expression *expression, + storage::v3::View view) + : ScanAll(input, output_symbol, view), + label_(label), + property_(property), + property_name_(property_name), + expression_(expression) { + DMG_ASSERT(expression, "Expression is not optional."); +} + +ACCEPT_WITH_INPUT(ScanAllByLabelPropertyValue) + +UniqueCursorPtr ScanAllByLabelPropertyValue::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ScanAllByLabelPropertyValueOperator); + + auto vertices = + [this](Frame &frame, ExecutionContext &context) -> std::optionalVertices( + view_, label_, property_, storage::v3::PropertyValue()))> { + auto *db = context.db_accessor; + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, view_); + auto value = expression_->Accept(evaluator); + if (value.IsNull()) return std::nullopt; + if (!value.IsPropertyValue()) { + throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type()); + } + return std::make_optional(db->Vertices(view_, label_, property_, storage::v3::PropertyValue(value))); + }; + return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), + std::move(vertices), "ScanAllByLabelPropertyValue"); +} + +ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr &input, Symbol output_symbol, + storage::v3::LabelId label, storage::v3::PropertyId property, + const std::string &property_name, storage::v3::View view) + : ScanAll(input, output_symbol, view), label_(label), property_(property), property_name_(property_name) {} + +ACCEPT_WITH_INPUT(ScanAllByLabelProperty) + +UniqueCursorPtr ScanAllByLabelProperty::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ScanAllByLabelPropertyOperator); + + auto vertices = [this](Frame &frame, ExecutionContext &context) { + auto *db = context.db_accessor; + return std::make_optional(db->Vertices(view_, label_, property_)); + }; + return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), + std::move(vertices), "ScanAllByLabelProperty"); +} + +ScanAllById::ScanAllById(const std::shared_ptr &input, Symbol output_symbol, Expression *expression, + storage::v3::View view) + : ScanAll(input, output_symbol, view), expression_(expression) { + MG_ASSERT(expression); +} + +ACCEPT_WITH_INPUT(ScanAllById) + +UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ScanAllByIdOperator); + + auto vertices = [this](Frame &frame, ExecutionContext &context) -> std::optional> { + auto *db = context.db_accessor; + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, view_); + auto value = expression_->Accept(evaluator); + if (!value.IsNumeric()) return std::nullopt; + int64_t id = value.IsInt() ? value.ValueInt() : value.ValueDouble(); + if (value.IsDouble() && id != value.ValueDouble()) return std::nullopt; + auto maybe_vertex = db->FindVertex(storage::v3::Gid::FromInt(id), view_); + if (!maybe_vertex) return std::nullopt; + return std::vector{*maybe_vertex}; + }; + return MakeUniqueCursorPtr>(mem, output_symbol_, input_->MakeCursor(mem), + std::move(vertices), "ScanAllById"); +} + +namespace { +bool CheckExistingNode(const VertexAccessor &new_node, const Symbol &existing_node_sym, Frame &frame) { + const TypedValue &existing_node = frame[existing_node_sym]; + if (existing_node.IsNull()) return false; + ExpectType(existing_node_sym, existing_node, TypedValue::Type::Vertex); + return existing_node.ValueVertex() == new_node; +} + +template +auto UnwrapEdgesResult(storage::v3::Result &&result) { + if (result.HasError()) { + switch (result.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to get relationships of a deleted node."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get relationships from a node that doesn't exist."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when accessing relationships."); + } + } + return std::move(*result); +} + +} // namespace + +Expand::Expand(const std::shared_ptr &input, Symbol input_symbol, Symbol node_symbol, + Symbol edge_symbol, EdgeAtom::Direction direction, + const std::vector &edge_types, bool existing_node, storage::v3::View view) + : input_(input ? input : std::make_shared()), + input_symbol_(input_symbol), + common_{node_symbol, edge_symbol, direction, edge_types, existing_node}, + view_(view) {} + +ACCEPT_WITH_INPUT(Expand) + +UniqueCursorPtr Expand::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ExpandOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Expand::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.emplace_back(common_.node_symbol); + symbols.emplace_back(common_.edge_symbol); + return symbols; +} + +Expand::ExpandCursor::ExpandCursor(const Expand &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +bool Expand::ExpandCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Expand"); + + // A helper function for expanding a node from an edge. + auto pull_node = [this, &frame](const EdgeAccessor &new_edge, EdgeAtom::Direction direction) { + if (self_.common_.existing_node) return; + switch (direction) { + case EdgeAtom::Direction::IN: + frame[self_.common_.node_symbol] = new_edge.From(); + break; + case EdgeAtom::Direction::OUT: + frame[self_.common_.node_symbol] = new_edge.To(); + break; + case EdgeAtom::Direction::BOTH: + LOG_FATAL("Must indicate exact expansion direction here"); + } + }; + + while (true) { + if (MustAbort(context)) throw HintedAbortError(); + // attempt to get a value from the incoming edges + if (in_edges_ && *in_edges_it_ != in_edges_->end()) { + auto edge = *(*in_edges_it_)++; + frame[self_.common_.edge_symbol] = edge; + pull_node(edge, EdgeAtom::Direction::IN); + return true; + } + + // attempt to get a value from the outgoing edges + if (out_edges_ && *out_edges_it_ != out_edges_->end()) { + auto edge = *(*out_edges_it_)++; + // when expanding in EdgeAtom::Direction::BOTH directions + // we should do only one expansion for cycles, and it was + // already done in the block above + if (self_.common_.direction == EdgeAtom::Direction::BOTH && edge.IsCycle()) continue; + frame[self_.common_.edge_symbol] = edge; + pull_node(edge, EdgeAtom::Direction::OUT); + return true; + } + + // If we are here, either the edges have not been initialized, + // or they have been exhausted. Attempt to initialize the edges. + if (!InitEdges(frame, context)) return false; + + // we have re-initialized the edges, continue with the loop + } +} + +void Expand::ExpandCursor::Shutdown() { input_cursor_->Shutdown(); } + +void Expand::ExpandCursor::Reset() { + input_cursor_->Reset(); + in_edges_ = std::nullopt; + in_edges_it_ = std::nullopt; + out_edges_ = std::nullopt; + out_edges_it_ = std::nullopt; +} + +bool Expand::ExpandCursor::InitEdges(Frame &frame, ExecutionContext &context) { + // Input Vertex could be null if it is created by a failed optional match. In + // those cases we skip that input pull and continue with the next. + while (true) { + if (!input_cursor_->Pull(frame, context)) return false; + TypedValue &vertex_value = frame[self_.input_symbol_]; + + // Null check due to possible failed optional match. + if (vertex_value.IsNull()) continue; + + ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex); + auto &vertex = vertex_value.ValueVertex(); + + auto direction = self_.common_.direction; + if (direction == EdgeAtom::Direction::IN || direction == EdgeAtom::Direction::BOTH) { + if (self_.common_.existing_node) { + TypedValue &existing_node = frame[self_.common_.node_symbol]; + // old_node_value may be Null when using optional matching + if (!existing_node.IsNull()) { + ExpectType(self_.common_.node_symbol, existing_node, TypedValue::Type::Vertex); + in_edges_.emplace( + UnwrapEdgesResult(vertex.InEdges(self_.view_, self_.common_.edge_types, existing_node.ValueVertex()))); + } + } else { + in_edges_.emplace(UnwrapEdgesResult(vertex.InEdges(self_.view_, self_.common_.edge_types))); + } + if (in_edges_) { + in_edges_it_.emplace(in_edges_->begin()); + } + } + + if (direction == EdgeAtom::Direction::OUT || direction == EdgeAtom::Direction::BOTH) { + if (self_.common_.existing_node) { + TypedValue &existing_node = frame[self_.common_.node_symbol]; + // old_node_value may be Null when using optional matching + if (!existing_node.IsNull()) { + ExpectType(self_.common_.node_symbol, existing_node, TypedValue::Type::Vertex); + out_edges_.emplace( + UnwrapEdgesResult(vertex.OutEdges(self_.view_, self_.common_.edge_types, existing_node.ValueVertex()))); + } + } else { + out_edges_.emplace(UnwrapEdgesResult(vertex.OutEdges(self_.view_, self_.common_.edge_types))); + } + if (out_edges_) { + out_edges_it_.emplace(out_edges_->begin()); + } + } + + return true; + } +} + +ExpandVariable::ExpandVariable(const std::shared_ptr &input, Symbol input_symbol, Symbol node_symbol, + Symbol edge_symbol, EdgeAtom::Type type, EdgeAtom::Direction direction, + const std::vector &edge_types, bool is_reverse, + Expression *lower_bound, Expression *upper_bound, bool existing_node, + ExpansionLambda filter_lambda, std::optional weight_lambda, + std::optional total_weight) + : input_(input ? input : std::make_shared()), + input_symbol_(input_symbol), + common_{node_symbol, edge_symbol, direction, edge_types, existing_node}, + type_(type), + is_reverse_(is_reverse), + lower_bound_(lower_bound), + upper_bound_(upper_bound), + filter_lambda_(filter_lambda), + weight_lambda_(weight_lambda), + total_weight_(total_weight) { + DMG_ASSERT(type_ == EdgeAtom::Type::DEPTH_FIRST || type_ == EdgeAtom::Type::BREADTH_FIRST || + type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH, + "ExpandVariable can only be used with breadth first, depth first or " + "weighted shortest path type"); + DMG_ASSERT(!(type_ == EdgeAtom::Type::BREADTH_FIRST && is_reverse), "Breadth first expansion can't be reversed"); +} + +ACCEPT_WITH_INPUT(ExpandVariable) + +std::vector ExpandVariable::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.emplace_back(common_.node_symbol); + symbols.emplace_back(common_.edge_symbol); + return symbols; +} + +namespace { + +/** + * Helper function that returns an iterable over + * pairs + * for the given params. + * + * @param vertex - The vertex to expand from. + * @param direction - Expansion direction. All directions (IN, OUT, BOTH) + * are supported. + * @param memory - Used to allocate the result. + * @return See above. + */ +auto ExpandFromVertex(const VertexAccessor &vertex, EdgeAtom::Direction direction, + const std::vector &edge_types, utils::MemoryResource *memory) { + // wraps an EdgeAccessor into a pair + auto wrapper = [](EdgeAtom::Direction direction, auto &&edges) { + return iter::imap([direction](const auto &edge) { return std::make_pair(edge, direction); }, + std::forward(edges)); + }; + + storage::v3::View view = storage::v3::View::OLD; + utils::pmr::vector chain_elements(memory); + + if (direction != EdgeAtom::Direction::OUT) { + auto edges = UnwrapEdgesResult(vertex.InEdges(view, edge_types)); + if (edges.begin() != edges.end()) { + chain_elements.emplace_back(wrapper(EdgeAtom::Direction::IN, std::move(edges))); + } + } + if (direction != EdgeAtom::Direction::IN) { + auto edges = UnwrapEdgesResult(vertex.OutEdges(view, edge_types)); + if (edges.begin() != edges.end()) { + chain_elements.emplace_back(wrapper(EdgeAtom::Direction::OUT, std::move(edges))); + } + } + + // TODO: Investigate whether itertools perform heap allocation? + return iter::chain.from_iterable(std::move(chain_elements)); +} + +} // namespace + +class ExpandVariableCursor : public Cursor { + public: + ExpandVariableCursor(const ExpandVariable &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)), edges_(mem), edges_it_(mem) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("ExpandVariable"); + + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + while (true) { + if (Expand(frame, context)) return true; + + if (PullInput(frame, context)) { + // if lower bound is zero we also yield empty paths + if (lower_bound_ == 0) { + auto &start_vertex = frame[self_.input_symbol_].ValueVertex(); + if (!self_.common_.existing_node) { + frame[self_.common_.node_symbol] = start_vertex; + return true; + } else if (CheckExistingNode(start_vertex, self_.common_.node_symbol, frame)) { + return true; + } + } + // if lower bound is not zero, we just continue, the next + // loop iteration will attempt to expand and we're good + } else + return false; + // else continue with the loop, try to expand again + // because we succesfully pulled from the input + } + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + edges_.clear(); + edges_it_.clear(); + } + + private: + const ExpandVariable &self_; + const UniqueCursorPtr input_cursor_; + // bounds. in the cursor they are not optional but set to + // default values if missing in the ExpandVariable operator + // initialize to arbitrary values, they should only be used + // after a successful pull from the input + int64_t upper_bound_{-1}; + int64_t lower_bound_{-1}; + + // a stack of edge iterables corresponding to the level/depth of + // the expansion currently being Pulled + using ExpandEdges = decltype(ExpandFromVertex(std::declval(), EdgeAtom::Direction::IN, + self_.common_.edge_types, utils::NewDeleteResource())); + + utils::pmr::vector edges_; + // an iterator indicating the position in the corresponding edges_ element + utils::pmr::vectorbegin())> edges_it_; + + /** + * Helper function that Pulls from the input vertex and + * makes iteration over it's edges possible. + * + * @return If the Pull succeeded. If not, this VariableExpandCursor + * is exhausted. + */ + bool PullInput(Frame &frame, ExecutionContext &context) { + // Input Vertex could be null if it is created by a failed optional match. + // In those cases we skip that input pull and continue with the next. + while (true) { + if (MustAbort(context)) throw HintedAbortError(); + if (!input_cursor_->Pull(frame, context)) return false; + TypedValue &vertex_value = frame[self_.input_symbol_]; + + // Null check due to possible failed optional match. + if (vertex_value.IsNull()) continue; + + ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex); + auto &vertex = vertex_value.ValueVertex(); + + // Evaluate the upper and lower bounds. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + auto calc_bound = [&evaluator](auto &bound) { + auto value = EvaluateInt(&evaluator, bound, "Variable expansion bound"); + if (value < 0) throw QueryRuntimeException("Variable expansion bound must be a non-negative integer."); + return value; + }; + + lower_bound_ = self_.lower_bound_ ? calc_bound(self_.lower_bound_) : 1; + upper_bound_ = self_.upper_bound_ ? calc_bound(self_.upper_bound_) : std::numeric_limits::max(); + + if (upper_bound_ > 0) { + auto *memory = edges_.get_allocator().GetMemoryResource(); + edges_.emplace_back(ExpandFromVertex(vertex, self_.common_.direction, self_.common_.edge_types, memory)); + edges_it_.emplace_back(edges_.back().begin()); + } + + // reset the frame value to an empty edge list + auto *pull_memory = context.evaluation_context.memory; + frame[self_.common_.edge_symbol] = TypedValue::TVector(pull_memory); + + return true; + } + } + + // Helper function for appending an edge to the list on the frame. + void AppendEdge(const EdgeAccessor &new_edge, utils::pmr::vector *edges_on_frame) { + // We are placing an edge on the frame. It is possible that there already + // exists an edge on the frame for this level. If so first remove it. + DMG_ASSERT(edges_.size() > 0, "Edges are empty"); + if (self_.is_reverse_) { + // TODO: This is innefficient, we should look into replacing + // vector with something else for TypedValue::List. + size_t diff = edges_on_frame->size() - std::min(edges_on_frame->size(), edges_.size() - 1U); + if (diff > 0U) edges_on_frame->erase(edges_on_frame->begin(), edges_on_frame->begin() + diff); + edges_on_frame->emplace(edges_on_frame->begin(), new_edge); + } else { + edges_on_frame->resize(std::min(edges_on_frame->size(), edges_.size() - 1U)); + edges_on_frame->emplace_back(new_edge); + } + } + + /** + * Performs a single expansion for the current state of this + * VariableExpansionCursor. + * + * @return True if the expansion was a success and this Cursor's + * consumer can consume it. False if the expansion failed. In that + * case no more expansions are available from the current input + * vertex and another Pull from the input cursor should be performed. + */ + bool Expand(Frame &frame, ExecutionContext &context) { + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + // Some expansions might not be valid due to edge uniqueness and + // existing_node criterions, so expand in a loop until either the input + // vertex is exhausted or a valid variable-length expansion is available. + while (true) { + if (MustAbort(context)) throw HintedAbortError(); + // pop from the stack while there is stuff to pop and the current + // level is exhausted + while (!edges_.empty() && edges_it_.back() == edges_.back().end()) { + edges_.pop_back(); + edges_it_.pop_back(); + } + + // check if we exhausted everything, if so return false + if (edges_.empty()) return false; + + // we use this a lot + auto &edges_on_frame = frame[self_.common_.edge_symbol].ValueList(); + + // it is possible that edges_on_frame does not contain as many + // elements as edges_ due to edge-uniqueness (when a whole layer + // gets exhausted but no edges are valid). for that reason only + // pop from edges_on_frame if they contain enough elements + if (self_.is_reverse_) { + auto diff = edges_on_frame.size() - std::min(edges_on_frame.size(), edges_.size()); + if (diff > 0) { + edges_on_frame.erase(edges_on_frame.begin(), edges_on_frame.begin() + diff); + } + } else { + edges_on_frame.resize(std::min(edges_on_frame.size(), edges_.size())); + } + + // if we are here, we have a valid stack, + // get the edge, increase the relevant iterator + auto current_edge = *edges_it_.back()++; + + // Check edge-uniqueness. + bool found_existing = + std::any_of(edges_on_frame.begin(), edges_on_frame.end(), + [¤t_edge](const TypedValue &edge) { return current_edge.first == edge.ValueEdge(); }); + if (found_existing) continue; + + AppendEdge(current_edge.first, &edges_on_frame); + VertexAccessor current_vertex = + current_edge.second == EdgeAtom::Direction::IN ? current_edge.first.From() : current_edge.first.To(); + + if (!self_.common_.existing_node) { + frame[self_.common_.node_symbol] = current_vertex; + } + + // Skip expanding out of filtered expansion. + frame[self_.filter_lambda_.inner_edge_symbol] = current_edge.first; + frame[self_.filter_lambda_.inner_node_symbol] = current_vertex; + if (self_.filter_lambda_.expression && !EvaluateFilter(evaluator, self_.filter_lambda_.expression)) continue; + + // we are doing depth-first search, so place the current + // edge's expansions onto the stack, if we should continue to expand + if (upper_bound_ > static_cast(edges_.size())) { + auto *memory = edges_.get_allocator().GetMemoryResource(); + edges_.emplace_back( + ExpandFromVertex(current_vertex, self_.common_.direction, self_.common_.edge_types, memory)); + edges_it_.emplace_back(edges_.back().begin()); + } + + if (self_.common_.existing_node && !CheckExistingNode(current_vertex, self_.common_.node_symbol, frame)) continue; + + // We only yield true if we satisfy the lower bound. + if (static_cast(edges_on_frame.size()) >= lower_bound_) + return true; + else + continue; + } + } +}; + +class STShortestPathCursor : public query::v2::plan::Cursor { + public: + STShortestPathCursor(const ExpandVariable &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input()->MakeCursor(mem)) { + MG_ASSERT(self_.common_.existing_node, + "s-t shortest path algorithm should only " + "be used when `existing_node` flag is " + "set!"); + } + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("STShortestPath"); + + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + while (input_cursor_->Pull(frame, context)) { + const auto &source_tv = frame[self_.input_symbol_]; + const auto &sink_tv = frame[self_.common_.node_symbol]; + + // It is possible that source or sink vertex is Null due to optional + // matching. + if (source_tv.IsNull() || sink_tv.IsNull()) continue; + + const auto &source = source_tv.ValueVertex(); + const auto &sink = sink_tv.ValueVertex(); + + int64_t lower_bound = + self_.lower_bound_ ? EvaluateInt(&evaluator, self_.lower_bound_, "Min depth in breadth-first expansion") : 1; + int64_t upper_bound = self_.upper_bound_ + ? EvaluateInt(&evaluator, self_.upper_bound_, "Max depth in breadth-first expansion") + : std::numeric_limits::max(); + + if (upper_bound < 1 || lower_bound > upper_bound) continue; + + if (FindPath(*context.db_accessor, source, sink, lower_bound, upper_bound, &frame, &evaluator, context)) { + return true; + } + } + return false; + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { input_cursor_->Reset(); } + + private: + const ExpandVariable &self_; + UniqueCursorPtr input_cursor_; + + using VertexEdgeMapT = utils::pmr::unordered_map>; + + void ReconstructPath(const VertexAccessor &midpoint, const VertexEdgeMapT &in_edge, const VertexEdgeMapT &out_edge, + Frame *frame, utils::MemoryResource *pull_memory) { + utils::pmr::vector result(pull_memory); + auto last_vertex = midpoint; + while (true) { + const auto &last_edge = in_edge.at(last_vertex); + if (!last_edge) break; + last_vertex = last_edge->From() == last_vertex ? last_edge->To() : last_edge->From(); + result.emplace_back(*last_edge); + } + std::reverse(result.begin(), result.end()); + last_vertex = midpoint; + while (true) { + const auto &last_edge = out_edge.at(last_vertex); + if (!last_edge) break; + last_vertex = last_edge->From() == last_vertex ? last_edge->To() : last_edge->From(); + result.emplace_back(*last_edge); + } + frame->at(self_.common_.edge_symbol) = std::move(result); + } + + bool ShouldExpand(const VertexAccessor &vertex, const EdgeAccessor &edge, Frame *frame, + ExpressionEvaluator *evaluator) { + if (!self_.filter_lambda_.expression) return true; + + frame->at(self_.filter_lambda_.inner_node_symbol) = vertex; + frame->at(self_.filter_lambda_.inner_edge_symbol) = edge; + + TypedValue result = self_.filter_lambda_.expression->Accept(*evaluator); + if (result.IsNull()) return false; + if (result.IsBool()) return result.ValueBool(); + + throw QueryRuntimeException("Expansion condition must evaluate to boolean or null"); + } + + bool FindPath(const DbAccessor &dba, const VertexAccessor &source, const VertexAccessor &sink, int64_t lower_bound, + int64_t upper_bound, Frame *frame, ExpressionEvaluator *evaluator, const ExecutionContext &context) { + using utils::Contains; + + if (source == sink) return false; + + // We expand from both directions, both from the source and the sink. + // Expansions meet at the middle of the path if it exists. This should + // perform better for real-world like graphs where the expansion front + // grows exponentially, effectively reducing the exponent by half. + + auto *pull_memory = evaluator->GetMemoryResource(); + // Holds vertices at the current level of expansion from the source + // (sink). + utils::pmr::vector source_frontier(pull_memory); + utils::pmr::vector sink_frontier(pull_memory); + + // Holds vertices we can expand to from `source_frontier` + // (`sink_frontier`). + utils::pmr::vector source_next(pull_memory); + utils::pmr::vector sink_next(pull_memory); + + // Maps each vertex we visited expanding from the source (sink) to the + // edge used. Necessary for path reconstruction. + VertexEdgeMapT in_edge(pull_memory); + VertexEdgeMapT out_edge(pull_memory); + + size_t current_length = 0; + + source_frontier.emplace_back(source); + in_edge[source] = std::nullopt; + sink_frontier.emplace_back(sink); + out_edge[sink] = std::nullopt; + + while (true) { + if (MustAbort(context)) throw HintedAbortError(); + // Top-down step (expansion from the source). + ++current_length; + if (current_length > upper_bound) return false; + + for (const auto &vertex : source_frontier) { + if (self_.common_.direction != EdgeAtom::Direction::IN) { + auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::v3::View::OLD, self_.common_.edge_types)); + for (const auto &edge : out_edges) { + if (ShouldExpand(edge.To(), edge, frame, evaluator) && !Contains(in_edge, edge.To())) { + in_edge.emplace(edge.To(), edge); + if (Contains(out_edge, edge.To())) { + if (current_length >= lower_bound) { + ReconstructPath(edge.To(), in_edge, out_edge, frame, pull_memory); + return true; + } else { + return false; + } + } + source_next.push_back(edge.To()); + } + } + } + if (self_.common_.direction != EdgeAtom::Direction::OUT) { + auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::v3::View::OLD, self_.common_.edge_types)); + for (const auto &edge : in_edges) { + if (ShouldExpand(edge.From(), edge, frame, evaluator) && !Contains(in_edge, edge.From())) { + in_edge.emplace(edge.From(), edge); + if (Contains(out_edge, edge.From())) { + if (current_length >= lower_bound) { + ReconstructPath(edge.From(), in_edge, out_edge, frame, pull_memory); + return true; + } else { + return false; + } + } + source_next.push_back(edge.From()); + } + } + } + } + + if (source_next.empty()) return false; + source_frontier.clear(); + std::swap(source_frontier, source_next); + + // Bottom-up step (expansion from the sink). + ++current_length; + if (current_length > upper_bound) return false; + + // When expanding from the sink we have to be careful which edge + // endpoint we pass to `should_expand`, because everything is + // reversed. + for (const auto &vertex : sink_frontier) { + if (self_.common_.direction != EdgeAtom::Direction::OUT) { + auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::v3::View::OLD, self_.common_.edge_types)); + for (const auto &edge : out_edges) { + if (ShouldExpand(vertex, edge, frame, evaluator) && !Contains(out_edge, edge.To())) { + out_edge.emplace(edge.To(), edge); + if (Contains(in_edge, edge.To())) { + if (current_length >= lower_bound) { + ReconstructPath(edge.To(), in_edge, out_edge, frame, pull_memory); + return true; + } else { + return false; + } + } + sink_next.push_back(edge.To()); + } + } + } + if (self_.common_.direction != EdgeAtom::Direction::IN) { + auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::v3::View::OLD, self_.common_.edge_types)); + for (const auto &edge : in_edges) { + if (ShouldExpand(vertex, edge, frame, evaluator) && !Contains(out_edge, edge.From())) { + out_edge.emplace(edge.From(), edge); + if (Contains(in_edge, edge.From())) { + if (current_length >= lower_bound) { + ReconstructPath(edge.From(), in_edge, out_edge, frame, pull_memory); + return true; + } else { + return false; + } + } + sink_next.push_back(edge.From()); + } + } + } + } + + if (sink_next.empty()) return false; + sink_frontier.clear(); + std::swap(sink_frontier, sink_next); + } + } +}; + +class SingleSourceShortestPathCursor : public query::v2::plan::Cursor { + public: + SingleSourceShortestPathCursor(const ExpandVariable &self, utils::MemoryResource *mem) + : self_(self), + input_cursor_(self_.input()->MakeCursor(mem)), + processed_(mem), + to_visit_current_(mem), + to_visit_next_(mem) { + MG_ASSERT(!self_.common_.existing_node, + "Single source shortest path algorithm " + "should not be used when `existing_node` " + "flag is set, s-t shortest path algorithm " + "should be used instead!"); + } + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("SingleSourceShortestPath"); + + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + + // for the given (edge, vertex) pair checks if they satisfy the + // "where" condition. if so, places them in the to_visit_ structure. + auto expand_pair = [this, &evaluator, &frame](EdgeAccessor edge, VertexAccessor vertex) { + // if we already processed the given vertex it doesn't get expanded + if (processed_.find(vertex) != processed_.end()) return; + + frame[self_.filter_lambda_.inner_edge_symbol] = edge; + frame[self_.filter_lambda_.inner_node_symbol] = vertex; + + if (self_.filter_lambda_.expression) { + TypedValue result = self_.filter_lambda_.expression->Accept(evaluator); + switch (result.type()) { + case TypedValue::Type::Null: + return; + case TypedValue::Type::Bool: + if (!result.ValueBool()) return; + break; + default: + throw QueryRuntimeException("Expansion condition must evaluate to boolean or null."); + } + } + to_visit_next_.emplace_back(edge, vertex); + processed_.emplace(vertex, edge); + }; + + // populates the to_visit_next_ structure with expansions + // from the given vertex. skips expansions that don't satisfy + // the "where" condition. + auto expand_from_vertex = [this, &expand_pair](const auto &vertex) { + if (self_.common_.direction != EdgeAtom::Direction::IN) { + auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::v3::View::OLD, self_.common_.edge_types)); + for (const auto &edge : out_edges) expand_pair(edge, edge.To()); + } + if (self_.common_.direction != EdgeAtom::Direction::OUT) { + auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::v3::View::OLD, self_.common_.edge_types)); + for (const auto &edge : in_edges) expand_pair(edge, edge.From()); + } + }; + + // do it all in a loop because we skip some elements + while (true) { + if (MustAbort(context)) throw HintedAbortError(); + // if we have nothing to visit on the current depth, switch to next + if (to_visit_current_.empty()) to_visit_current_.swap(to_visit_next_); + + // if current is still empty, it means both are empty, so pull from + // input + if (to_visit_current_.empty()) { + if (!input_cursor_->Pull(frame, context)) return false; + + to_visit_current_.clear(); + to_visit_next_.clear(); + processed_.clear(); + + const auto &vertex_value = frame[self_.input_symbol_]; + // it is possible that the vertex is Null due to optional matching + if (vertex_value.IsNull()) continue; + lower_bound_ = self_.lower_bound_ + ? EvaluateInt(&evaluator, self_.lower_bound_, "Min depth in breadth-first expansion") + : 1; + upper_bound_ = self_.upper_bound_ + ? EvaluateInt(&evaluator, self_.upper_bound_, "Max depth in breadth-first expansion") + : std::numeric_limits::max(); + + if (upper_bound_ < 1 || lower_bound_ > upper_bound_) continue; + + const auto &vertex = vertex_value.ValueVertex(); + processed_.emplace(vertex, std::nullopt); + expand_from_vertex(vertex); + + // go back to loop start and see if we expanded anything + continue; + } + + // take the next expansion from the queue + auto expansion = to_visit_current_.back(); + to_visit_current_.pop_back(); + + // create the frame value for the edges + auto *pull_memory = context.evaluation_context.memory; + utils::pmr::vector edge_list(pull_memory); + edge_list.emplace_back(expansion.first); + auto last_vertex = expansion.second; + while (true) { + const EdgeAccessor &last_edge = edge_list.back().ValueEdge(); + last_vertex = last_edge.From() == last_vertex ? last_edge.To() : last_edge.From(); + // origin_vertex must be in processed + const auto &previous_edge = processed_.find(last_vertex)->second; + if (!previous_edge) break; + + edge_list.emplace_back(previous_edge.value()); + } + + // expand only if what we've just expanded is less then max depth + if (static_cast(edge_list.size()) < upper_bound_) expand_from_vertex(expansion.second); + + if (static_cast(edge_list.size()) < lower_bound_) continue; + + frame[self_.common_.node_symbol] = expansion.second; + + // place edges on the frame in the correct order + std::reverse(edge_list.begin(), edge_list.end()); + frame[self_.common_.edge_symbol] = std::move(edge_list); + + return true; + } + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + processed_.clear(); + to_visit_next_.clear(); + to_visit_current_.clear(); + } + + private: + const ExpandVariable &self_; + const UniqueCursorPtr input_cursor_; + + // Depth bounds. Calculated on each pull from the input, the initial value + // is irrelevant. + int64_t lower_bound_{-1}; + int64_t upper_bound_{-1}; + + // maps vertices to the edge they got expanded from. it is an optional + // edge because the root does not get expanded from anything. + // contains visited vertices as well as those scheduled to be visited. + utils::pmr::unordered_map> processed_; + // edge/vertex pairs we have yet to visit, for current and next depth + utils::pmr::vector> to_visit_current_; + utils::pmr::vector> to_visit_next_; +}; + +class ExpandWeightedShortestPathCursor : public query::v2::plan::Cursor { + public: + ExpandWeightedShortestPathCursor(const ExpandVariable &self, utils::MemoryResource *mem) + : self_(self), + input_cursor_(self_.input_->MakeCursor(mem)), + total_cost_(mem), + previous_(mem), + yielded_vertices_(mem), + pq_(mem) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("ExpandWeightedShortestPath"); + + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + auto create_state = [this](const VertexAccessor &vertex, int64_t depth) { + return std::make_pair(vertex, upper_bound_set_ ? depth : 0); + }; + + // For the given (edge, vertex, weight, depth) tuple checks if they + // satisfy the "where" condition. if so, places them in the priority + // queue. + auto expand_pair = [this, &evaluator, &frame, &create_state](const EdgeAccessor &edge, const VertexAccessor &vertex, + const TypedValue &total_weight, int64_t depth) { + auto *memory = evaluator.GetMemoryResource(); + if (self_.filter_lambda_.expression) { + frame[self_.filter_lambda_.inner_edge_symbol] = edge; + frame[self_.filter_lambda_.inner_node_symbol] = vertex; + + if (!EvaluateFilter(evaluator, self_.filter_lambda_.expression)) return; + } + + frame[self_.weight_lambda_->inner_edge_symbol] = edge; + frame[self_.weight_lambda_->inner_node_symbol] = vertex; + + TypedValue current_weight = self_.weight_lambda_->expression->Accept(evaluator); + + if (!current_weight.IsNumeric() && !current_weight.IsDuration()) { + throw QueryRuntimeException("Calculated weight must be numeric or a Duration, got {}.", current_weight.type()); + } + + const auto is_valid_numeric = [&] { + return current_weight.IsNumeric() && (current_weight >= TypedValue(0, memory)).ValueBool(); + }; + + const auto is_valid_duration = [&] { + return current_weight.IsDuration() && (current_weight >= TypedValue(utils::Duration(0), memory)).ValueBool(); + }; + + if (!is_valid_numeric() && !is_valid_duration()) { + throw QueryRuntimeException("Calculated weight must be non-negative!"); + } + + auto next_state = create_state(vertex, depth); + + TypedValue next_weight = std::invoke([&] { + if (total_weight.IsNull()) { + return current_weight; + } + + ValidateWeightTypes(current_weight, total_weight); + + return TypedValue(current_weight, memory) + total_weight; + }); + + auto found_it = total_cost_.find(next_state); + if (found_it != total_cost_.end() && (found_it->second.IsNull() || (found_it->second <= next_weight).ValueBool())) + return; + + pq_.push({next_weight, depth + 1, vertex, edge}); + }; + + // Populates the priority queue structure with expansions + // from the given vertex. skips expansions that don't satisfy + // the "where" condition. + auto expand_from_vertex = [this, &expand_pair](const VertexAccessor &vertex, const TypedValue &weight, + int64_t depth) { + if (self_.common_.direction != EdgeAtom::Direction::IN) { + auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::v3::View::OLD, self_.common_.edge_types)); + for (const auto &edge : out_edges) { + expand_pair(edge, edge.To(), weight, depth); + } + } + if (self_.common_.direction != EdgeAtom::Direction::OUT) { + auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::v3::View::OLD, self_.common_.edge_types)); + for (const auto &edge : in_edges) { + expand_pair(edge, edge.From(), weight, depth); + } + } + }; + + while (true) { + if (MustAbort(context)) throw HintedAbortError(); + if (pq_.empty()) { + if (!input_cursor_->Pull(frame, context)) return false; + const auto &vertex_value = frame[self_.input_symbol_]; + if (vertex_value.IsNull()) continue; + auto vertex = vertex_value.ValueVertex(); + if (self_.common_.existing_node) { + const auto &node = frame[self_.common_.node_symbol]; + // Due to optional matching the existing node could be null. + // Skip expansion for such nodes. + if (node.IsNull()) continue; + } + if (self_.upper_bound_) { + upper_bound_ = EvaluateInt(&evaluator, self_.upper_bound_, "Max depth in weighted shortest path expansion"); + upper_bound_set_ = true; + } else { + upper_bound_ = std::numeric_limits::max(); + upper_bound_set_ = false; + } + if (upper_bound_ < 1) + throw QueryRuntimeException( + "Maximum depth in weighted shortest path expansion must be at " + "least 1."); + + // Clear existing data structures. + previous_.clear(); + total_cost_.clear(); + yielded_vertices_.clear(); + + pq_.push({TypedValue(), 0, vertex, std::nullopt}); + // We are adding the starting vertex to the set of yielded vertices + // because we don't want to yield paths that end with the starting + // vertex. + yielded_vertices_.insert(vertex); + } + + while (!pq_.empty()) { + if (MustAbort(context)) throw HintedAbortError(); + auto [current_weight, current_depth, current_vertex, current_edge] = pq_.top(); + pq_.pop(); + + auto current_state = create_state(current_vertex, current_depth); + + // Check if the vertex has already been processed. + if (total_cost_.find(current_state) != total_cost_.end()) { + continue; + } + previous_.emplace(current_state, current_edge); + total_cost_.emplace(current_state, current_weight); + + // Expand only if what we've just expanded is less than max depth. + if (current_depth < upper_bound_) expand_from_vertex(current_vertex, current_weight, current_depth); + + // If we yielded a path for a vertex already, make the expansion but + // don't return the path again. + if (yielded_vertices_.find(current_vertex) != yielded_vertices_.end()) continue; + + // Reconstruct the path. + auto last_vertex = current_vertex; + auto last_depth = current_depth; + auto *pull_memory = context.evaluation_context.memory; + utils::pmr::vector edge_list(pull_memory); + while (true) { + // Origin_vertex must be in previous. + const auto &previous_edge = previous_.find(create_state(last_vertex, last_depth))->second; + if (!previous_edge) break; + last_vertex = previous_edge->From() == last_vertex ? previous_edge->To() : previous_edge->From(); + last_depth--; + edge_list.emplace_back(previous_edge.value()); + } + + // Place destination node on the frame, handle existence flag. + if (self_.common_.existing_node) { + const auto &node = frame[self_.common_.node_symbol]; + if ((node != TypedValue(current_vertex, pull_memory)).ValueBool()) + continue; + else + // Prevent expanding other paths, because we found the + // shortest to existing node. + ClearQueue(); + } else { + frame[self_.common_.node_symbol] = current_vertex; + } + + if (!self_.is_reverse_) { + // Place edges on the frame in the correct order. + std::reverse(edge_list.begin(), edge_list.end()); + } + frame[self_.common_.edge_symbol] = std::move(edge_list); + frame[self_.total_weight_.value()] = current_weight; + yielded_vertices_.insert(current_vertex); + return true; + } + } + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + previous_.clear(); + total_cost_.clear(); + yielded_vertices_.clear(); + ClearQueue(); + } + + private: + const ExpandVariable &self_; + const UniqueCursorPtr input_cursor_; + + // Upper bound on the path length. + int64_t upper_bound_{-1}; + bool upper_bound_set_{false}; + + struct WspStateHash { + size_t operator()(const std::pair &key) const { + return utils::HashCombine{}(key.first, key.second); + } + }; + + // Maps vertices to weights they got in expansion. + utils::pmr::unordered_map, TypedValue, WspStateHash> total_cost_; + + // Maps vertices to edges used to reach them. + utils::pmr::unordered_map, std::optional, WspStateHash> previous_; + + // Keeps track of vertices for which we yielded a path already. + utils::pmr::unordered_set yielded_vertices_; + + static void ValidateWeightTypes(const TypedValue &lhs, const TypedValue &rhs) { + if (!((lhs.IsNumeric() && lhs.IsNumeric()) || (rhs.IsDuration() && rhs.IsDuration()))) { + throw QueryRuntimeException(utils::MessageWithLink( + "All weights should be of the same type, either numeric or a Duration. Please update the weight " + "expression or the filter expression.", + "https://memgr.ph/wsp")); + } + } + + // Priority queue comparator. Keep lowest weight on top of the queue. + class PriorityQueueComparator { + public: + bool operator()(const std::tuple> &lhs, + const std::tuple> &rhs) { + const auto &lhs_weight = std::get<0>(lhs); + const auto &rhs_weight = std::get<0>(rhs); + // Null defines minimum value for all types + if (lhs_weight.IsNull()) { + return false; + } + + if (rhs_weight.IsNull()) { + return true; + } + + ValidateWeightTypes(lhs_weight, rhs_weight); + return (lhs_weight > rhs_weight).ValueBool(); + } + }; + + std::priority_queue>, + utils::pmr::vector>>, + PriorityQueueComparator> + pq_; + + void ClearQueue() { + while (!pq_.empty()) pq_.pop(); + } +}; + +UniqueCursorPtr ExpandVariable::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ExpandVariableOperator); + + switch (type_) { + case EdgeAtom::Type::BREADTH_FIRST: + if (common_.existing_node) { + return MakeUniqueCursorPtr(mem, *this, mem); + } else { + return MakeUniqueCursorPtr(mem, *this, mem); + } + case EdgeAtom::Type::DEPTH_FIRST: + return MakeUniqueCursorPtr(mem, *this, mem); + case EdgeAtom::Type::WEIGHTED_SHORTEST_PATH: + return MakeUniqueCursorPtr(mem, *this, mem); + case EdgeAtom::Type::SINGLE: + LOG_FATAL("ExpandVariable should not be planned for a single expansion!"); + } +} + +class ConstructNamedPathCursor : public Cursor { + public: + ConstructNamedPathCursor(const ConstructNamedPath &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input()->MakeCursor(mem)) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("ConstructNamedPath"); + + if (!input_cursor_->Pull(frame, context)) return false; + + auto symbol_it = self_.path_elements_.begin(); + DMG_ASSERT(symbol_it != self_.path_elements_.end(), "Named path must contain at least one node"); + + const auto &start_vertex = frame[*symbol_it++]; + auto *pull_memory = context.evaluation_context.memory; + // In an OPTIONAL MATCH everything could be Null. + if (start_vertex.IsNull()) { + frame[self_.path_symbol_] = TypedValue(pull_memory); + return true; + } + + DMG_ASSERT(start_vertex.IsVertex(), "First named path element must be a vertex"); + query::v2::Path path(start_vertex.ValueVertex(), pull_memory); + + // If the last path element symbol was for an edge list, then + // the next symbol is a vertex and it should not append to the path + // because + // expansion already did it. + bool last_was_edge_list = false; + + for (; symbol_it != self_.path_elements_.end(); symbol_it++) { + const auto &expansion = frame[*symbol_it]; + // We can have Null (OPTIONAL MATCH), a vertex, an edge, or an edge + // list (variable expand or BFS). + switch (expansion.type()) { + case TypedValue::Type::Null: + frame[self_.path_symbol_] = TypedValue(pull_memory); + return true; + case TypedValue::Type::Vertex: + if (!last_was_edge_list) path.Expand(expansion.ValueVertex()); + last_was_edge_list = false; + break; + case TypedValue::Type::Edge: + path.Expand(expansion.ValueEdge()); + break; + case TypedValue::Type::List: { + last_was_edge_list = true; + // We need to expand all edges in the list and intermediary + // vertices. + const auto &edges = expansion.ValueList(); + for (const auto &edge_value : edges) { + const auto &edge = edge_value.ValueEdge(); + const auto &from = edge.From(); + if (path.vertices().back() == from) + path.Expand(edge, edge.To()); + else + path.Expand(edge, from); + } + break; + } + default: + LOG_FATAL("Unsupported type in named path construction"); + + break; + } + } + + frame[self_.path_symbol_] = path; + return true; + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { input_cursor_->Reset(); } + + private: + const ConstructNamedPath self_; + const UniqueCursorPtr input_cursor_; +}; + +ACCEPT_WITH_INPUT(ConstructNamedPath) + +UniqueCursorPtr ConstructNamedPath::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ConstructNamedPathOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector ConstructNamedPath::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.emplace_back(path_symbol_); + return symbols; +} + +Filter::Filter(const std::shared_ptr &input, Expression *expression) + : input_(input ? input : std::make_shared()), expression_(expression) {} + +ACCEPT_WITH_INPUT(Filter) + +UniqueCursorPtr Filter::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::FilterOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Filter::ModifiedSymbols(const SymbolTable &table) const { return input_->ModifiedSymbols(table); } + +Filter::FilterCursor::FilterCursor(const Filter &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input_->MakeCursor(mem)) {} + +bool Filter::FilterCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Filter"); + + // Like all filters, newly set values should not affect filtering of old + // nodes and edges. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + while (input_cursor_->Pull(frame, context)) { + if (EvaluateFilter(evaluator, self_.expression_)) return true; + } + return false; +} + +void Filter::FilterCursor::Shutdown() { input_cursor_->Shutdown(); } + +void Filter::FilterCursor::Reset() { input_cursor_->Reset(); } + +Produce::Produce(const std::shared_ptr &input, const std::vector &named_expressions) + : input_(input ? input : std::make_shared()), named_expressions_(named_expressions) {} + +ACCEPT_WITH_INPUT(Produce) + +UniqueCursorPtr Produce::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ProduceOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Produce::OutputSymbols(const SymbolTable &symbol_table) const { + std::vector symbols; + for (const auto &named_expr : named_expressions_) { + symbols.emplace_back(symbol_table.at(*named_expr)); + } + return symbols; +} + +std::vector Produce::ModifiedSymbols(const SymbolTable &table) const { return OutputSymbols(table); } + +Produce::ProduceCursor::ProduceCursor(const Produce &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input_->MakeCursor(mem)) {} + +bool Produce::ProduceCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Produce"); + + if (input_cursor_->Pull(frame, context)) { + // Produce should always yield the latest results. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::NEW); + for (auto named_expr : self_.named_expressions_) named_expr->Accept(evaluator); + + return true; + } + return false; +} + +void Produce::ProduceCursor::Shutdown() { input_cursor_->Shutdown(); } + +void Produce::ProduceCursor::Reset() { input_cursor_->Reset(); } + +Delete::Delete(const std::shared_ptr &input_, const std::vector &expressions, + bool detach_) + : input_(input_), expressions_(expressions), detach_(detach_) {} + +ACCEPT_WITH_INPUT(Delete) + +UniqueCursorPtr Delete::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::DeleteOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Delete::ModifiedSymbols(const SymbolTable &table) const { return input_->ModifiedSymbols(table); } + +Delete::DeleteCursor::DeleteCursor(const Delete &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input_->MakeCursor(mem)) {} + +bool Delete::DeleteCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Delete"); + + if (!input_cursor_->Pull(frame, context)) return false; + + // Delete should get the latest information, this way it is also possible + // to delete newly added nodes and edges. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::NEW); + auto *pull_memory = context.evaluation_context.memory; + // collect expressions results so edges can get deleted before vertices + // this is necessary because an edge that gets deleted could block vertex + // deletion + utils::pmr::vector expression_results(pull_memory); + expression_results.reserve(self_.expressions_.size()); + for (Expression *expression : self_.expressions_) { + expression_results.emplace_back(expression->Accept(evaluator)); + } + + auto &dba = *context.db_accessor; + // delete edges first + for (TypedValue &expression_result : expression_results) { + if (MustAbort(context)) throw HintedAbortError(); + if (expression_result.type() == TypedValue::Type::Edge) { + auto maybe_value = dba.RemoveEdge(&expression_result.ValueEdge()); + if (maybe_value.HasError()) { + switch (maybe_value.GetError()) { + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::DELETED_OBJECT: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when deleting an edge."); + } + } + context.execution_stats[ExecutionStats::Key::DELETED_EDGES] += 1; + if (context.trigger_context_collector && maybe_value.GetValue()) { + context.trigger_context_collector->RegisterDeletedObject(*maybe_value.GetValue()); + } + } + } + + // delete vertices + for (TypedValue &expression_result : expression_results) { + if (MustAbort(context)) throw HintedAbortError(); + switch (expression_result.type()) { + case TypedValue::Type::Vertex: { + auto &va = expression_result.ValueVertex(); + if (self_.detach_) { + auto res = dba.DetachRemoveVertex(&va); + if (res.HasError()) { + switch (res.GetError()) { + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::DELETED_OBJECT: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when deleting a node."); + } + } + + context.execution_stats[ExecutionStats::Key::DELETED_NODES] += 1; + if (*res) { + context.execution_stats[ExecutionStats::Key::DELETED_EDGES] += static_cast((*res)->second.size()); + } + std::invoke([&] { + if (!context.trigger_context_collector || !*res) { + return; + } + + context.trigger_context_collector->RegisterDeletedObject((*res)->first); + if (!context.trigger_context_collector->ShouldRegisterDeletedObject()) { + return; + } + for (const auto &edge : (*res)->second) { + context.trigger_context_collector->RegisterDeletedObject(edge); + } + }); + } else { + auto res = dba.RemoveVertex(&va); + if (res.HasError()) { + switch (res.GetError()) { + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::VERTEX_HAS_EDGES: + throw RemoveAttachedVertexException(); + case storage::v3::Error::DELETED_OBJECT: + case storage::v3::Error::PROPERTIES_DISABLED: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when deleting a node."); + } + } + context.execution_stats[ExecutionStats::Key::DELETED_NODES] += 1; + if (context.trigger_context_collector && res.GetValue()) { + context.trigger_context_collector->RegisterDeletedObject(*res.GetValue()); + } + } + break; + } + + // skip Edges (already deleted) and Nulls (can occur in optional + // match) + case TypedValue::Type::Edge: + case TypedValue::Type::Null: + break; + // check we're not trying to delete anything except vertices and edges + default: + throw QueryRuntimeException("Only edges and vertices can be deleted."); + } + } + + return true; +} + +void Delete::DeleteCursor::Shutdown() { input_cursor_->Shutdown(); } + +void Delete::DeleteCursor::Reset() { input_cursor_->Reset(); } + +SetProperty::SetProperty(const std::shared_ptr &input, storage::v3::PropertyId property, + PropertyLookup *lhs, Expression *rhs) + : input_(input), property_(property), lhs_(lhs), rhs_(rhs) {} + +ACCEPT_WITH_INPUT(SetProperty) + +UniqueCursorPtr SetProperty::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::SetPropertyOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector SetProperty::ModifiedSymbols(const SymbolTable &table) const { + return input_->ModifiedSymbols(table); +} + +SetProperty::SetPropertyCursor::SetPropertyCursor(const SetProperty &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +bool SetProperty::SetPropertyCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("SetProperty"); + + if (!input_cursor_->Pull(frame, context)) return false; + + // Set, just like Create needs to see the latest changes. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::NEW); + TypedValue lhs = self_.lhs_->expression_->Accept(evaluator); + TypedValue rhs = self_.rhs_->Accept(evaluator); + + switch (lhs.type()) { + case TypedValue::Type::Vertex: { + auto old_value = PropsSetChecked(&lhs.ValueVertex(), self_.property_, rhs); + context.execution_stats[ExecutionStats::Key::UPDATED_PROPERTIES] += 1; + if (context.trigger_context_collector) { + // rhs cannot be moved because it was created with the allocator that is only valid during current pull + context.trigger_context_collector->RegisterSetObjectProperty(lhs.ValueVertex(), self_.property_, + TypedValue{std::move(old_value)}, TypedValue{rhs}); + } + break; + } + case TypedValue::Type::Edge: { + auto old_value = PropsSetChecked(&lhs.ValueEdge(), self_.property_, rhs); + context.execution_stats[ExecutionStats::Key::UPDATED_PROPERTIES] += 1; + if (context.trigger_context_collector) { + // rhs cannot be moved because it was created with the allocator that is only valid during current pull + context.trigger_context_collector->RegisterSetObjectProperty(lhs.ValueEdge(), self_.property_, + TypedValue{std::move(old_value)}, TypedValue{rhs}); + } + break; + } + case TypedValue::Type::Null: + // Skip setting properties on Null (can occur in optional match). + break; + case TypedValue::Type::Map: + // Semantically modifying a map makes sense, but it's not supported due + // to all the copying we do (when PropertyValue -> TypedValue and in + // ExpressionEvaluator). So even though we set a map property here, that + // is never visible to the user and it's not stored. + // TODO: fix above described bug + default: + throw QueryRuntimeException("Properties can only be set on edges and vertices."); + } + return true; +} + +void SetProperty::SetPropertyCursor::Shutdown() { input_cursor_->Shutdown(); } + +void SetProperty::SetPropertyCursor::Reset() { input_cursor_->Reset(); } + +SetProperties::SetProperties(const std::shared_ptr &input, Symbol input_symbol, Expression *rhs, Op op) + : input_(input), input_symbol_(input_symbol), rhs_(rhs), op_(op) {} + +ACCEPT_WITH_INPUT(SetProperties) + +UniqueCursorPtr SetProperties::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::SetPropertiesOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector SetProperties::ModifiedSymbols(const SymbolTable &table) const { + return input_->ModifiedSymbols(table); +} + +SetProperties::SetPropertiesCursor::SetPropertiesCursor(const SetProperties &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +namespace { + +template +concept AccessorWithProperties = requires(T value, storage::v3::PropertyId property_id, + storage::v3::PropertyValue property_value) { + { + value.ClearProperties() + } -> std::same_as>>; + {value.SetProperty(property_id, property_value)}; +}; + +/// Helper function that sets the given values on either a Vertex or an Edge. +/// +/// @tparam TRecordAccessor Either RecordAccessor or +/// RecordAccessor +template +void SetPropertiesOnRecord(TRecordAccessor *record, const TypedValue &rhs, SetProperties::Op op, + ExecutionContext *context) { + std::optional> old_values; + const bool should_register_change = + context->trigger_context_collector && + context->trigger_context_collector->ShouldRegisterObjectPropertyChange(); + if (op == SetProperties::Op::REPLACE) { + auto maybe_value = record->ClearProperties(); + if (maybe_value.HasError()) { + switch (maybe_value.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to set properties on a deleted graph element."); + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Can't set property because properties on edges are disabled."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when setting properties."); + } + } + + if (should_register_change) { + old_values.emplace(std::move(*maybe_value)); + } + } + + auto get_props = [](const auto &record) { + auto maybe_props = record.Properties(storage::v3::View::NEW); + if (maybe_props.HasError()) { + switch (maybe_props.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to get properties from a deleted object."); + case storage::v3::Error::NONEXISTENT_OBJECT: + throw query::v2::QueryRuntimeException("Trying to get properties from an object that doesn't exist."); + case storage::v3::Error::SERIALIZATION_ERROR: + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Unexpected error when getting properties."); + } + } + return *maybe_props; + }; + + auto register_set_property = [&](auto &&returned_old_value, auto key, auto &&new_value) { + auto old_value = [&]() -> storage::v3::PropertyValue { + if (!old_values) { + return std::forward(returned_old_value); + } + + if (auto it = old_values->find(key); it != old_values->end()) { + return std::move(it->second); + } + + return {}; + }(); + + context->trigger_context_collector->RegisterSetObjectProperty( + *record, key, TypedValue(std::move(old_value)), TypedValue(std::forward(new_value))); + }; + + auto set_props = [&, record](auto properties) { + for (auto &kv : properties) { + auto maybe_error = record->SetProperty(kv.first, kv.second); + if (maybe_error.HasError()) { + switch (maybe_error.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to set properties on a deleted graph element."); + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException("Can't set property because properties on edges are disabled."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when setting properties."); + } + } + + if (should_register_change) { + register_set_property(std::move(*maybe_error), kv.first, std::move(kv.second)); + } + } + }; + + switch (rhs.type()) { + case TypedValue::Type::Edge: + set_props(get_props(rhs.ValueEdge())); + break; + case TypedValue::Type::Vertex: + set_props(get_props(rhs.ValueVertex())); + break; + case TypedValue::Type::Map: { + for (const auto &kv : rhs.ValueMap()) { + auto key = context->db_accessor->NameToProperty(kv.first); + auto old_value = PropsSetChecked(record, key, kv.second); + if (should_register_change) { + register_set_property(std::move(old_value), key, kv.second); + } + } + break; + } + default: + throw QueryRuntimeException( + "Right-hand side in SET expression must be a node, an edge or a " + "map."); + } + + if (should_register_change && old_values) { + // register removed properties + for (auto &[property_id, property_value] : *old_values) { + context->trigger_context_collector->RegisterRemovedObjectProperty(*record, property_id, + TypedValue(std::move(property_value))); + } + } +} + +} // namespace + +bool SetProperties::SetPropertiesCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("SetProperties"); + + if (!input_cursor_->Pull(frame, context)) return false; + + TypedValue &lhs = frame[self_.input_symbol_]; + + // Set, just like Create needs to see the latest changes. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::NEW); + TypedValue rhs = self_.rhs_->Accept(evaluator); + + switch (lhs.type()) { + case TypedValue::Type::Vertex: + SetPropertiesOnRecord(&lhs.ValueVertex(), rhs, self_.op_, &context); + break; + case TypedValue::Type::Edge: + SetPropertiesOnRecord(&lhs.ValueEdge(), rhs, self_.op_, &context); + break; + case TypedValue::Type::Null: + // Skip setting properties on Null (can occur in optional match). + break; + default: + throw QueryRuntimeException("Properties can only be set on edges and vertices."); + } + return true; +} + +void SetProperties::SetPropertiesCursor::Shutdown() { input_cursor_->Shutdown(); } + +void SetProperties::SetPropertiesCursor::Reset() { input_cursor_->Reset(); } + +SetLabels::SetLabels(const std::shared_ptr &input, Symbol input_symbol, + const std::vector &labels) + : input_(input), input_symbol_(input_symbol), labels_(labels) {} + +ACCEPT_WITH_INPUT(SetLabels) + +UniqueCursorPtr SetLabels::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::SetLabelsOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector SetLabels::ModifiedSymbols(const SymbolTable &table) const { + return input_->ModifiedSymbols(table); +} + +SetLabels::SetLabelsCursor::SetLabelsCursor(const SetLabels &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +bool SetLabels::SetLabelsCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("SetLabels"); + + if (!input_cursor_->Pull(frame, context)) return false; + + TypedValue &vertex_value = frame[self_.input_symbol_]; + // Skip setting labels on Null (can occur in optional match). + if (vertex_value.IsNull()) return true; + ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex); + auto &vertex = vertex_value.ValueVertex(); + for (auto label : self_.labels_) { + auto maybe_value = vertex.AddLabel(label); + if (maybe_value.HasError()) { + switch (maybe_value.GetError()) { + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to set a label on a deleted node."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when setting a label."); + } + } + + if (context.trigger_context_collector && *maybe_value) { + context.trigger_context_collector->RegisterSetVertexLabel(vertex, label); + } + } + + return true; +} + +void SetLabels::SetLabelsCursor::Shutdown() { input_cursor_->Shutdown(); } + +void SetLabels::SetLabelsCursor::Reset() { input_cursor_->Reset(); } + +RemoveProperty::RemoveProperty(const std::shared_ptr &input, storage::v3::PropertyId property, + PropertyLookup *lhs) + : input_(input), property_(property), lhs_(lhs) {} + +ACCEPT_WITH_INPUT(RemoveProperty) + +UniqueCursorPtr RemoveProperty::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::RemovePropertyOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector RemoveProperty::ModifiedSymbols(const SymbolTable &table) const { + return input_->ModifiedSymbols(table); +} + +RemoveProperty::RemovePropertyCursor::RemovePropertyCursor(const RemoveProperty &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +bool RemoveProperty::RemovePropertyCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("RemoveProperty"); + + if (!input_cursor_->Pull(frame, context)) return false; + + // Remove, just like Delete needs to see the latest changes. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::NEW); + TypedValue lhs = self_.lhs_->expression_->Accept(evaluator); + + auto remove_prop = [property = self_.property_, &context](auto *record) { + auto maybe_old_value = record->RemoveProperty(property); + if (maybe_old_value.HasError()) { + switch (maybe_old_value.GetError()) { + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to remove a property on a deleted graph element."); + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::PROPERTIES_DISABLED: + throw QueryRuntimeException( + "Can't remove property because properties on edges are " + "disabled."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when removing property."); + } + } + + if (context.trigger_context_collector) { + context.trigger_context_collector->RegisterRemovedObjectProperty(*record, property, + TypedValue(std::move(*maybe_old_value))); + } + }; + + switch (lhs.type()) { + case TypedValue::Type::Vertex: + remove_prop(&lhs.ValueVertex()); + break; + case TypedValue::Type::Edge: + remove_prop(&lhs.ValueEdge()); + break; + case TypedValue::Type::Null: + // Skip removing properties on Null (can occur in optional match). + break; + default: + throw QueryRuntimeException("Properties can only be removed from vertices and edges."); + } + return true; +} + +void RemoveProperty::RemovePropertyCursor::Shutdown() { input_cursor_->Shutdown(); } + +void RemoveProperty::RemovePropertyCursor::Reset() { input_cursor_->Reset(); } + +RemoveLabels::RemoveLabels(const std::shared_ptr &input, Symbol input_symbol, + const std::vector &labels) + : input_(input), input_symbol_(input_symbol), labels_(labels) {} + +ACCEPT_WITH_INPUT(RemoveLabels) + +UniqueCursorPtr RemoveLabels::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::RemoveLabelsOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector RemoveLabels::ModifiedSymbols(const SymbolTable &table) const { + return input_->ModifiedSymbols(table); +} + +RemoveLabels::RemoveLabelsCursor::RemoveLabelsCursor(const RemoveLabels &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +bool RemoveLabels::RemoveLabelsCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("RemoveLabels"); + + if (!input_cursor_->Pull(frame, context)) return false; + + TypedValue &vertex_value = frame[self_.input_symbol_]; + // Skip removing labels on Null (can occur in optional match). + if (vertex_value.IsNull()) return true; + ExpectType(self_.input_symbol_, vertex_value, TypedValue::Type::Vertex); + auto &vertex = vertex_value.ValueVertex(); + for (auto label : self_.labels_) { + auto maybe_value = vertex.RemoveLabel(label); + if (maybe_value.HasError()) { + switch (maybe_value.GetError()) { + case storage::v3::Error::SERIALIZATION_ERROR: + throw TransactionSerializationException(); + case storage::v3::Error::DELETED_OBJECT: + throw QueryRuntimeException("Trying to remove labels from a deleted node."); + case storage::v3::Error::VERTEX_HAS_EDGES: + case storage::v3::Error::PROPERTIES_DISABLED: + case storage::v3::Error::NONEXISTENT_OBJECT: + throw QueryRuntimeException("Unexpected error when removing labels from a node."); + } + } + + context.execution_stats[ExecutionStats::Key::DELETED_LABELS] += 1; + if (context.trigger_context_collector && *maybe_value) { + context.trigger_context_collector->RegisterRemovedVertexLabel(vertex, label); + } + } + + return true; +} + +void RemoveLabels::RemoveLabelsCursor::Shutdown() { input_cursor_->Shutdown(); } + +void RemoveLabels::RemoveLabelsCursor::Reset() { input_cursor_->Reset(); } + +EdgeUniquenessFilter::EdgeUniquenessFilter(const std::shared_ptr &input, Symbol expand_symbol, + const std::vector &previous_symbols) + : input_(input), expand_symbol_(expand_symbol), previous_symbols_(previous_symbols) {} + +ACCEPT_WITH_INPUT(EdgeUniquenessFilter) + +UniqueCursorPtr EdgeUniquenessFilter::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::EdgeUniquenessFilterOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector EdgeUniquenessFilter::ModifiedSymbols(const SymbolTable &table) const { + return input_->ModifiedSymbols(table); +} + +EdgeUniquenessFilter::EdgeUniquenessFilterCursor::EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &self, + utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)) {} + +namespace { +/** + * Returns true if: + * - a and b are either edge or edge-list values, and there + * is at least one matching edge in the two values + */ +bool ContainsSameEdge(const TypedValue &a, const TypedValue &b) { + auto compare_to_list = [](const TypedValue &list, const TypedValue &other) { + for (const TypedValue &list_elem : list.ValueList()) + if (ContainsSameEdge(list_elem, other)) return true; + return false; + }; + + if (a.type() == TypedValue::Type::List) return compare_to_list(a, b); + if (b.type() == TypedValue::Type::List) return compare_to_list(b, a); + + return a.ValueEdge() == b.ValueEdge(); +} +} // namespace + +bool EdgeUniquenessFilter::EdgeUniquenessFilterCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("EdgeUniquenessFilter"); + + auto expansion_ok = [&]() { + const auto &expand_value = frame[self_.expand_symbol_]; + for (const auto &previous_symbol : self_.previous_symbols_) { + const auto &previous_value = frame[previous_symbol]; + // This shouldn't raise a TypedValueException, because the planner + // makes sure these are all of the expected type. In case they are not + // an error should be raised long before this code is executed. + if (ContainsSameEdge(previous_value, expand_value)) return false; + } + return true; + }; + + while (input_cursor_->Pull(frame, context)) + if (expansion_ok()) return true; + return false; +} + +void EdgeUniquenessFilter::EdgeUniquenessFilterCursor::Shutdown() { input_cursor_->Shutdown(); } + +void EdgeUniquenessFilter::EdgeUniquenessFilterCursor::Reset() { input_cursor_->Reset(); } + +Accumulate::Accumulate(const std::shared_ptr &input, const std::vector &symbols, + bool advance_command) + : input_(input), symbols_(symbols), advance_command_(advance_command) {} + +ACCEPT_WITH_INPUT(Accumulate) + +std::vector Accumulate::ModifiedSymbols(const SymbolTable &) const { return symbols_; } + +class AccumulateCursor : public Cursor { + public: + AccumulateCursor(const Accumulate &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)), cache_(mem) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("Accumulate"); + + auto &dba = *context.db_accessor; + // cache all the input + if (!pulled_all_input_) { + while (input_cursor_->Pull(frame, context)) { + utils::pmr::vector row(cache_.get_allocator().GetMemoryResource()); + row.reserve(self_.symbols_.size()); + for (const Symbol &symbol : self_.symbols_) row.emplace_back(frame[symbol]); + cache_.emplace_back(std::move(row)); + } + pulled_all_input_ = true; + cache_it_ = cache_.begin(); + + if (self_.advance_command_) dba.AdvanceCommand(); + } + + if (MustAbort(context)) throw HintedAbortError(); + if (cache_it_ == cache_.end()) return false; + auto row_it = (cache_it_++)->begin(); + for (const Symbol &symbol : self_.symbols_) frame[symbol] = *row_it++; + return true; + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + cache_.clear(); + cache_it_ = cache_.begin(); + pulled_all_input_ = false; + } + + private: + const Accumulate &self_; + const UniqueCursorPtr input_cursor_; + utils::pmr::vector> cache_; + decltype(cache_.begin()) cache_it_ = cache_.begin(); + bool pulled_all_input_{false}; +}; + +UniqueCursorPtr Accumulate::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::AccumulateOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +Aggregate::Aggregate(const std::shared_ptr &input, const std::vector &aggregations, + const std::vector &group_by, const std::vector &remember) + : input_(input ? input : std::make_shared()), + aggregations_(aggregations), + group_by_(group_by), + remember_(remember) {} + +ACCEPT_WITH_INPUT(Aggregate) + +std::vector Aggregate::ModifiedSymbols(const SymbolTable &) const { + auto symbols = remember_; + for (const auto &elem : aggregations_) symbols.push_back(elem.output_sym); + return symbols; +} + +namespace { +/** Returns the default TypedValue for an Aggregation element. + * This value is valid both for returning when where are no inputs + * to the aggregation op, and for initializing an aggregation result + * when there are */ +TypedValue DefaultAggregationOpValue(const Aggregate::Element &element, utils::MemoryResource *memory) { + switch (element.op) { + case Aggregation::Op::COUNT: + return TypedValue(0, memory); + case Aggregation::Op::SUM: + case Aggregation::Op::MIN: + case Aggregation::Op::MAX: + case Aggregation::Op::AVG: + return TypedValue(memory); + case Aggregation::Op::COLLECT_LIST: + return TypedValue(TypedValue::TVector(memory)); + case Aggregation::Op::COLLECT_MAP: + return TypedValue(TypedValue::TMap(memory)); + } +} +} // namespace + +class AggregateCursor : public Cursor { + public: + AggregateCursor(const Aggregate &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input_->MakeCursor(mem)), aggregation_(mem) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("Aggregate"); + + if (!pulled_all_input_) { + ProcessAll(&frame, &context); + pulled_all_input_ = true; + aggregation_it_ = aggregation_.begin(); + + // in case there is no input and no group_bys we need to return true + // just this once + if (aggregation_.empty() && self_.group_by_.empty()) { + auto *pull_memory = context.evaluation_context.memory; + // place default aggregation values on the frame + for (const auto &elem : self_.aggregations_) + frame[elem.output_sym] = DefaultAggregationOpValue(elem, pull_memory); + // place null as remember values on the frame + for (const Symbol &remember_sym : self_.remember_) frame[remember_sym] = TypedValue(pull_memory); + return true; + } + } + + if (aggregation_it_ == aggregation_.end()) return false; + + // place aggregation values on the frame + auto aggregation_values_it = aggregation_it_->second.values_.begin(); + for (const auto &aggregation_elem : self_.aggregations_) + frame[aggregation_elem.output_sym] = *aggregation_values_it++; + + // place remember values on the frame + auto remember_values_it = aggregation_it_->second.remember_.begin(); + for (const Symbol &remember_sym : self_.remember_) frame[remember_sym] = *remember_values_it++; + + aggregation_it_++; + return true; + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + aggregation_.clear(); + aggregation_it_ = aggregation_.begin(); + pulled_all_input_ = false; + } + + private: + // Data structure for a single aggregation cache. + // Does NOT include the group-by values since those are a key in the + // aggregation map. The vectors in an AggregationValue contain one element for + // each aggregation in this LogicalOp. + struct AggregationValue { + explicit AggregationValue(utils::MemoryResource *mem) : counts_(mem), values_(mem), remember_(mem) {} + + // how many input rows have been aggregated in respective values_ element so + // far + // TODO: The counting value type should be changed to an unsigned type once + // TypedValue can support signed integer values larger than 64bits so that + // precision isn't lost. + utils::pmr::vector counts_; + // aggregated values. Initially Null (until at least one input row with a + // valid value gets processed) + utils::pmr::vector values_; + // remember values. + utils::pmr::vector remember_; + }; + + const Aggregate &self_; + const UniqueCursorPtr input_cursor_; + // storage for aggregated data + // map key is the vector of group-by values + // map value is an AggregationValue struct + utils::pmr::unordered_map, AggregationValue, + // use FNV collection hashing specialized for a + // vector of TypedValues + utils::FnvCollection, TypedValue, TypedValue::Hash>, + // custom equality + TypedValueVectorEqual> + aggregation_; + // iterator over the accumulated cache + decltype(aggregation_.begin()) aggregation_it_ = aggregation_.begin(); + // this LogicalOp pulls all from the input on it's first pull + // this switch tracks if this has been performed + bool pulled_all_input_{false}; + + /** + * Pulls from the input operator until exhausted and aggregates the + * results. If the input operator is not provided, a single call + * to ProcessOne is issued. + * + * Accumulation automatically groups the results so that `aggregation_` + * cache cardinality depends on number of + * aggregation results, and not on the number of inputs. + */ + void ProcessAll(Frame *frame, ExecutionContext *context) { + ExpressionEvaluator evaluator(frame, context->symbol_table, context->evaluation_context, context->db_accessor, + storage::v3::View::NEW); + while (input_cursor_->Pull(*frame, *context)) { + ProcessOne(*frame, &evaluator); + } + + // calculate AVG aggregations (so far they have only been summed) + for (size_t pos = 0; pos < self_.aggregations_.size(); ++pos) { + if (self_.aggregations_[pos].op != Aggregation::Op::AVG) continue; + for (auto &kv : aggregation_) { + AggregationValue &agg_value = kv.second; + auto count = agg_value.counts_[pos]; + auto *pull_memory = context->evaluation_context.memory; + if (count > 0) { + agg_value.values_[pos] = agg_value.values_[pos] / TypedValue(static_cast(count), pull_memory); + } + } + } + } + + /** + * Performs a single accumulation. + */ + void ProcessOne(const Frame &frame, ExpressionEvaluator *evaluator) { + auto *mem = aggregation_.get_allocator().GetMemoryResource(); + utils::pmr::vector group_by(mem); + group_by.reserve(self_.group_by_.size()); + for (Expression *expression : self_.group_by_) { + group_by.emplace_back(expression->Accept(*evaluator)); + } + auto &agg_value = aggregation_.try_emplace(std::move(group_by), mem).first->second; + EnsureInitialized(frame, &agg_value); + Update(evaluator, &agg_value); + } + + /** Ensures the new AggregationValue has been initialized. This means + * that the value vectors are filled with an appropriate number of Nulls, + * counts are set to 0 and remember values are remembered. + */ + void EnsureInitialized(const Frame &frame, AggregateCursor::AggregationValue *agg_value) const { + if (!agg_value->values_.empty()) return; + + for (const auto &agg_elem : self_.aggregations_) { + auto *mem = agg_value->values_.get_allocator().GetMemoryResource(); + agg_value->values_.emplace_back(DefaultAggregationOpValue(agg_elem, mem)); + } + agg_value->counts_.resize(self_.aggregations_.size(), 0); + + for (const Symbol &remember_sym : self_.remember_) agg_value->remember_.push_back(frame[remember_sym]); + } + + /** Updates the given AggregationValue with new data. Assumes that + * the AggregationValue has been initialized */ + void Update(ExpressionEvaluator *evaluator, AggregateCursor::AggregationValue *agg_value) { + DMG_ASSERT(self_.aggregations_.size() == agg_value->values_.size(), + "Expected as much AggregationValue.values_ as there are " + "aggregations."); + DMG_ASSERT(self_.aggregations_.size() == agg_value->counts_.size(), + "Expected as much AggregationValue.counts_ as there are " + "aggregations."); + + // we iterate over counts, values and aggregation info at the same time + auto count_it = agg_value->counts_.begin(); + auto value_it = agg_value->values_.begin(); + auto agg_elem_it = self_.aggregations_.begin(); + for (; count_it < agg_value->counts_.end(); count_it++, value_it++, agg_elem_it++) { + // COUNT(*) is the only case where input expression is optional + // handle it here + auto input_expr_ptr = agg_elem_it->value; + if (!input_expr_ptr) { + *count_it += 1; + *value_it = *count_it; + continue; + } + + TypedValue input_value = input_expr_ptr->Accept(*evaluator); + + // Aggregations skip Null input values. + if (input_value.IsNull()) continue; + const auto &agg_op = agg_elem_it->op; + *count_it += 1; + if (*count_it == 1) { + // first value, nothing to aggregate. check type, set and continue. + switch (agg_op) { + case Aggregation::Op::MIN: + case Aggregation::Op::MAX: + *value_it = input_value; + EnsureOkForMinMax(input_value); + break; + case Aggregation::Op::SUM: + case Aggregation::Op::AVG: + *value_it = input_value; + EnsureOkForAvgSum(input_value); + break; + case Aggregation::Op::COUNT: + *value_it = 1; + break; + case Aggregation::Op::COLLECT_LIST: + value_it->ValueList().push_back(input_value); + break; + case Aggregation::Op::COLLECT_MAP: + auto key = agg_elem_it->key->Accept(*evaluator); + if (key.type() != TypedValue::Type::String) throw QueryRuntimeException("Map key must be a string."); + value_it->ValueMap().emplace(key.ValueString(), input_value); + break; + } + continue; + } + + // aggregation of existing values + switch (agg_op) { + case Aggregation::Op::COUNT: + *value_it = *count_it; + break; + case Aggregation::Op::MIN: { + EnsureOkForMinMax(input_value); + try { + TypedValue comparison_result = input_value < *value_it; + // since we skip nulls we either have a valid comparison, or + // an exception was just thrown above + // safe to assume a bool TypedValue + if (comparison_result.ValueBool()) *value_it = input_value; + } catch (const TypedValueException &) { + throw QueryRuntimeException("Unable to get MIN of '{}' and '{}'.", input_value.type(), value_it->type()); + } + break; + } + case Aggregation::Op::MAX: { + // all comments as for Op::Min + EnsureOkForMinMax(input_value); + try { + TypedValue comparison_result = input_value > *value_it; + if (comparison_result.ValueBool()) *value_it = input_value; + } catch (const TypedValueException &) { + throw QueryRuntimeException("Unable to get MAX of '{}' and '{}'.", input_value.type(), value_it->type()); + } + break; + } + case Aggregation::Op::AVG: + // for averaging we sum first and divide by count once all + // the input has been processed + case Aggregation::Op::SUM: + EnsureOkForAvgSum(input_value); + *value_it = *value_it + input_value; + break; + case Aggregation::Op::COLLECT_LIST: + value_it->ValueList().push_back(input_value); + break; + case Aggregation::Op::COLLECT_MAP: + auto key = agg_elem_it->key->Accept(*evaluator); + if (key.type() != TypedValue::Type::String) throw QueryRuntimeException("Map key must be a string."); + value_it->ValueMap().emplace(key.ValueString(), input_value); + break; + } // end switch over Aggregation::Op enum + } // end loop over all aggregations + } + + /** Checks if the given TypedValue is legal in MIN and MAX. If not + * an appropriate exception is thrown. */ + void EnsureOkForMinMax(const TypedValue &value) const { + switch (value.type()) { + case TypedValue::Type::Bool: + case TypedValue::Type::Int: + case TypedValue::Type::Double: + case TypedValue::Type::String: + return; + default: + throw QueryRuntimeException( + "Only boolean, numeric and string values are allowed in " + "MIN and MAX aggregations."); + } + } + + /** Checks if the given TypedValue is legal in AVG and SUM. If not + * an appropriate exception is thrown. */ + void EnsureOkForAvgSum(const TypedValue &value) const { + switch (value.type()) { + case TypedValue::Type::Int: + case TypedValue::Type::Double: + return; + default: + throw QueryRuntimeException("Only numeric values allowed in SUM and AVG aggregations."); + } + } +}; + +UniqueCursorPtr Aggregate::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::AggregateOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +Skip::Skip(const std::shared_ptr &input, Expression *expression) + : input_(input), expression_(expression) {} + +ACCEPT_WITH_INPUT(Skip) + +UniqueCursorPtr Skip::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::SkipOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Skip::OutputSymbols(const SymbolTable &symbol_table) const { + // Propagate this to potential Produce. + return input_->OutputSymbols(symbol_table); +} + +std::vector Skip::ModifiedSymbols(const SymbolTable &table) const { return input_->ModifiedSymbols(table); } + +Skip::SkipCursor::SkipCursor(const Skip &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input_->MakeCursor(mem)) {} + +bool Skip::SkipCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Skip"); + + while (input_cursor_->Pull(frame, context)) { + if (to_skip_ == -1) { + // First successful pull from the input, evaluate the skip expression. + // The skip expression doesn't contain identifiers so graph view + // parameter is not important. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + TypedValue to_skip = self_.expression_->Accept(evaluator); + if (to_skip.type() != TypedValue::Type::Int) + throw QueryRuntimeException("Number of elements to skip must be an integer."); + + to_skip_ = to_skip.ValueInt(); + if (to_skip_ < 0) throw QueryRuntimeException("Number of elements to skip must be non-negative."); + } + + if (skipped_++ < to_skip_) continue; + return true; + } + return false; +} + +void Skip::SkipCursor::Shutdown() { input_cursor_->Shutdown(); } + +void Skip::SkipCursor::Reset() { + input_cursor_->Reset(); + to_skip_ = -1; + skipped_ = 0; +} + +Limit::Limit(const std::shared_ptr &input, Expression *expression) + : input_(input), expression_(expression) {} + +ACCEPT_WITH_INPUT(Limit) + +UniqueCursorPtr Limit::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::LimitOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Limit::OutputSymbols(const SymbolTable &symbol_table) const { + // Propagate this to potential Produce. + return input_->OutputSymbols(symbol_table); +} + +std::vector Limit::ModifiedSymbols(const SymbolTable &table) const { return input_->ModifiedSymbols(table); } + +Limit::LimitCursor::LimitCursor(const Limit &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input_->MakeCursor(mem)) {} + +bool Limit::LimitCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Limit"); + + // We need to evaluate the limit expression before the first input Pull + // because it might be 0 and thereby we shouldn't Pull from input at all. + // We can do this before Pulling from the input because the limit expression + // is not allowed to contain any identifiers. + if (limit_ == -1) { + // Limit expression doesn't contain identifiers so graph view is not + // important. + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + TypedValue limit = self_.expression_->Accept(evaluator); + if (limit.type() != TypedValue::Type::Int) + throw QueryRuntimeException("Limit on number of returned elements must be an integer."); + + limit_ = limit.ValueInt(); + if (limit_ < 0) throw QueryRuntimeException("Limit on number of returned elements must be non-negative."); + } + + // check we have not exceeded the limit before pulling + if (pulled_++ >= limit_) return false; + + return input_cursor_->Pull(frame, context); +} + +void Limit::LimitCursor::Shutdown() { input_cursor_->Shutdown(); } + +void Limit::LimitCursor::Reset() { + input_cursor_->Reset(); + limit_ = -1; + pulled_ = 0; +} + +OrderBy::OrderBy(const std::shared_ptr &input, const std::vector &order_by, + const std::vector &output_symbols) + : input_(input), output_symbols_(output_symbols) { + // split the order_by vector into two vectors of orderings and expressions + std::vector ordering; + ordering.reserve(order_by.size()); + order_by_.reserve(order_by.size()); + for (const auto &ordering_expression_pair : order_by) { + ordering.emplace_back(ordering_expression_pair.ordering); + order_by_.emplace_back(ordering_expression_pair.expression); + } + compare_ = TypedValueVectorCompare(ordering); +} + +ACCEPT_WITH_INPUT(OrderBy) + +std::vector OrderBy::OutputSymbols(const SymbolTable &symbol_table) const { + // Propagate this to potential Produce. + return input_->OutputSymbols(symbol_table); +} + +std::vector OrderBy::ModifiedSymbols(const SymbolTable &table) const { return input_->ModifiedSymbols(table); } + +class OrderByCursor : public Cursor { + public: + OrderByCursor(const OrderBy &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_.input_->MakeCursor(mem)), cache_(mem) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("OrderBy"); + + if (!did_pull_all_) { + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + auto *mem = cache_.get_allocator().GetMemoryResource(); + while (input_cursor_->Pull(frame, context)) { + // collect the order_by elements + utils::pmr::vector order_by(mem); + order_by.reserve(self_.order_by_.size()); + for (auto expression_ptr : self_.order_by_) { + order_by.emplace_back(expression_ptr->Accept(evaluator)); + } + + // collect the output elements + utils::pmr::vector output(mem); + output.reserve(self_.output_symbols_.size()); + for (const Symbol &output_sym : self_.output_symbols_) output.emplace_back(frame[output_sym]); + + cache_.push_back(Element{std::move(order_by), std::move(output)}); + } + + std::sort(cache_.begin(), cache_.end(), [this](const auto &pair1, const auto &pair2) { + return self_.compare_(pair1.order_by, pair2.order_by); + }); + + did_pull_all_ = true; + cache_it_ = cache_.begin(); + } + + if (cache_it_ == cache_.end()) return false; + + if (MustAbort(context)) throw HintedAbortError(); + + // place the output values on the frame + DMG_ASSERT(self_.output_symbols_.size() == cache_it_->remember.size(), + "Number of values does not match the number of output symbols " + "in OrderBy"); + auto output_sym_it = self_.output_symbols_.begin(); + for (const TypedValue &output : cache_it_->remember) frame[*output_sym_it++] = output; + + cache_it_++; + return true; + } + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + did_pull_all_ = false; + cache_.clear(); + cache_it_ = cache_.begin(); + } + + private: + struct Element { + utils::pmr::vector order_by; + utils::pmr::vector remember; + }; + + const OrderBy &self_; + const UniqueCursorPtr input_cursor_; + bool did_pull_all_{false}; + // a cache of elements pulled from the input + // the cache is filled and sorted (only on first elem) on first Pull + utils::pmr::vector cache_; + // iterator over the cache_, maintains state between Pulls + decltype(cache_.begin()) cache_it_ = cache_.begin(); +}; + +UniqueCursorPtr OrderBy::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::OrderByOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +Merge::Merge(const std::shared_ptr &input, const std::shared_ptr &merge_match, + const std::shared_ptr &merge_create) + : input_(input ? input : std::make_shared()), merge_match_(merge_match), merge_create_(merge_create) {} + +bool Merge::Accept(HierarchicalLogicalOperatorVisitor &visitor) { + if (visitor.PreVisit(*this)) { + input_->Accept(visitor) && merge_match_->Accept(visitor) && merge_create_->Accept(visitor); + } + return visitor.PostVisit(*this); +} + +UniqueCursorPtr Merge::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::MergeOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Merge::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + // Match and create branches should have the same symbols, so just take one + // of them. + auto my_symbols = merge_match_->OutputSymbols(table); + symbols.insert(symbols.end(), my_symbols.begin(), my_symbols.end()); + return symbols; +} + +Merge::MergeCursor::MergeCursor(const Merge &self, utils::MemoryResource *mem) + : input_cursor_(self.input_->MakeCursor(mem)), + merge_match_cursor_(self.merge_match_->MakeCursor(mem)), + merge_create_cursor_(self.merge_create_->MakeCursor(mem)) {} + +bool Merge::MergeCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Merge"); + + while (true) { + if (pull_input_) { + if (input_cursor_->Pull(frame, context)) { + // after a successful input from the input + // reset merge_match (it's expand iterators maintain state) + // and merge_create (could have a Once at the beginning) + merge_match_cursor_->Reset(); + merge_create_cursor_->Reset(); + } else + // input is exhausted, we're done + return false; + } + + // pull from the merge_match cursor + if (merge_match_cursor_->Pull(frame, context)) { + // if successful, next Pull from this should not pull_input_ + pull_input_ = false; + return true; + } else { + // failed to Pull from the merge_match cursor + if (pull_input_) { + // if we have just now pulled from the input + // and failed to pull from merge_match, we should create + __attribute__((unused)) bool merge_create_pull_result = merge_create_cursor_->Pull(frame, context); + DMG_ASSERT(merge_create_pull_result, "MergeCreate must never fail"); + return true; + } + // We have exhausted merge_match_cursor_ after 1 or more successful + // Pulls. Attempt next input_cursor_ pull + pull_input_ = true; + continue; + } + } +} + +void Merge::MergeCursor::Shutdown() { + input_cursor_->Shutdown(); + merge_match_cursor_->Shutdown(); + merge_create_cursor_->Shutdown(); +} + +void Merge::MergeCursor::Reset() { + input_cursor_->Reset(); + merge_match_cursor_->Reset(); + merge_create_cursor_->Reset(); + pull_input_ = true; +} + +Optional::Optional(const std::shared_ptr &input, const std::shared_ptr &optional, + const std::vector &optional_symbols) + : input_(input ? input : std::make_shared()), optional_(optional), optional_symbols_(optional_symbols) {} + +bool Optional::Accept(HierarchicalLogicalOperatorVisitor &visitor) { + if (visitor.PreVisit(*this)) { + input_->Accept(visitor) && optional_->Accept(visitor); + } + return visitor.PostVisit(*this); +} + +UniqueCursorPtr Optional::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::OptionalOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Optional::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + auto my_symbols = optional_->ModifiedSymbols(table); + symbols.insert(symbols.end(), my_symbols.begin(), my_symbols.end()); + return symbols; +} + +Optional::OptionalCursor::OptionalCursor(const Optional &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)), optional_cursor_(self.optional_->MakeCursor(mem)) {} + +bool Optional::OptionalCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Optional"); + + while (true) { + if (pull_input_) { + if (input_cursor_->Pull(frame, context)) { + // after a successful input from the input + // reset optional_ (it's expand iterators maintain state) + optional_cursor_->Reset(); + } else + // input is exhausted, we're done + return false; + } + + // pull from the optional_ cursor + if (optional_cursor_->Pull(frame, context)) { + // if successful, next Pull from this should not pull_input_ + pull_input_ = false; + return true; + } else { + // failed to Pull from the merge_match cursor + if (pull_input_) { + // if we have just now pulled from the input + // and failed to pull from optional_ so set the + // optional symbols to Null, ensure next time the + // input gets pulled and return true + for (const Symbol &sym : self_.optional_symbols_) frame[sym] = TypedValue(context.evaluation_context.memory); + pull_input_ = true; + return true; + } + // we have exhausted optional_cursor_ after 1 or more successful Pulls + // attempt next input_cursor_ pull + pull_input_ = true; + continue; + } + } +} + +void Optional::OptionalCursor::Shutdown() { + input_cursor_->Shutdown(); + optional_cursor_->Shutdown(); +} + +void Optional::OptionalCursor::Reset() { + input_cursor_->Reset(); + optional_cursor_->Reset(); + pull_input_ = true; +} + +Unwind::Unwind(const std::shared_ptr &input, Expression *input_expression, Symbol output_symbol) + : input_(input ? input : std::make_shared()), + input_expression_(input_expression), + output_symbol_(output_symbol) {} + +ACCEPT_WITH_INPUT(Unwind) + +std::vector Unwind::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.emplace_back(output_symbol_); + return symbols; +} + +class UnwindCursor : public Cursor { + public: + UnwindCursor(const Unwind &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)), input_value_(mem) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("Unwind"); + while (true) { + if (MustAbort(context)) throw HintedAbortError(); + // if we reached the end of our list of values + // pull from the input + if (input_value_it_ == input_value_.end()) { + if (!input_cursor_->Pull(frame, context)) return false; + + // successful pull from input, initialize value and iterator + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::OLD); + TypedValue input_value = self_.input_expression_->Accept(evaluator); + if (input_value.type() != TypedValue::Type::List) + throw QueryRuntimeException("Argument of UNWIND must be a list, but '{}' was provided.", input_value.type()); + // Copy the evaluted input_value_list to our vector. + input_value_ = input_value.ValueList(); + input_value_it_ = input_value_.begin(); + } + + // if we reached the end of our list of values goto back to top + if (input_value_it_ == input_value_.end()) continue; + + frame[self_.output_symbol_] = *input_value_it_++; + return true; + } + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + input_value_.clear(); + input_value_it_ = input_value_.end(); + } + + private: + const Unwind &self_; + const UniqueCursorPtr input_cursor_; + // typed values we are unwinding and yielding + utils::pmr::vector input_value_; + // current position in input_value_ + decltype(input_value_)::iterator input_value_it_ = input_value_.end(); +}; + +UniqueCursorPtr Unwind::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::UnwindOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +class DistinctCursor : public Cursor { + public: + DistinctCursor(const Distinct &self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self.input_->MakeCursor(mem)), seen_rows_(mem) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("Distinct"); + + while (true) { + if (!input_cursor_->Pull(frame, context)) return false; + + utils::pmr::vector row(seen_rows_.get_allocator().GetMemoryResource()); + row.reserve(self_.value_symbols_.size()); + for (const auto &symbol : self_.value_symbols_) row.emplace_back(frame[symbol]); + if (seen_rows_.insert(std::move(row)).second) return true; + } + } + + void Shutdown() override { input_cursor_->Shutdown(); } + + void Reset() override { + input_cursor_->Reset(); + seen_rows_.clear(); + } + + private: + const Distinct &self_; + const UniqueCursorPtr input_cursor_; + // a set of already seen rows + utils::pmr::unordered_set, + // use FNV collection hashing specialized for a + // vector of TypedValue + utils::FnvCollection, TypedValue, TypedValue::Hash>, + TypedValueVectorEqual> + seen_rows_; +}; + +Distinct::Distinct(const std::shared_ptr &input, const std::vector &value_symbols) + : input_(input ? input : std::make_shared()), value_symbols_(value_symbols) {} + +ACCEPT_WITH_INPUT(Distinct) + +UniqueCursorPtr Distinct::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::DistinctOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Distinct::OutputSymbols(const SymbolTable &symbol_table) const { + // Propagate this to potential Produce. + return input_->OutputSymbols(symbol_table); +} + +std::vector Distinct::ModifiedSymbols(const SymbolTable &table) const { return input_->ModifiedSymbols(table); } + +Union::Union(const std::shared_ptr &left_op, const std::shared_ptr &right_op, + const std::vector &union_symbols, const std::vector &left_symbols, + const std::vector &right_symbols) + : left_op_(left_op), + right_op_(right_op), + union_symbols_(union_symbols), + left_symbols_(left_symbols), + right_symbols_(right_symbols) {} + +UniqueCursorPtr Union::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::UnionOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +bool Union::Accept(HierarchicalLogicalOperatorVisitor &visitor) { + if (visitor.PreVisit(*this)) { + if (left_op_->Accept(visitor)) { + right_op_->Accept(visitor); + } + } + return visitor.PostVisit(*this); +} + +std::vector Union::OutputSymbols(const SymbolTable &) const { return union_symbols_; } + +std::vector Union::ModifiedSymbols(const SymbolTable &) const { return union_symbols_; } + +WITHOUT_SINGLE_INPUT(Union); + +Union::UnionCursor::UnionCursor(const Union &self, utils::MemoryResource *mem) + : self_(self), left_cursor_(self.left_op_->MakeCursor(mem)), right_cursor_(self.right_op_->MakeCursor(mem)) {} + +bool Union::UnionCursor::Pull(Frame &frame, ExecutionContext &context) { + SCOPED_PROFILE_OP("Union"); + + utils::pmr::unordered_map results(context.evaluation_context.memory); + if (left_cursor_->Pull(frame, context)) { + // collect values from the left child + for (const auto &output_symbol : self_.left_symbols_) { + results[output_symbol.name()] = frame[output_symbol]; + } + } else if (right_cursor_->Pull(frame, context)) { + // collect values from the right child + for (const auto &output_symbol : self_.right_symbols_) { + results[output_symbol.name()] = frame[output_symbol]; + } + } else { + return false; + } + + // put collected values on frame under union symbols + for (const auto &symbol : self_.union_symbols_) { + frame[symbol] = results[symbol.name()]; + } + return true; +} + +void Union::UnionCursor::Shutdown() { + left_cursor_->Shutdown(); + right_cursor_->Shutdown(); +} + +void Union::UnionCursor::Reset() { + left_cursor_->Reset(); + right_cursor_->Reset(); +} + +std::vector Cartesian::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = left_op_->ModifiedSymbols(table); + auto right = right_op_->ModifiedSymbols(table); + symbols.insert(symbols.end(), right.begin(), right.end()); + return symbols; +} + +bool Cartesian::Accept(HierarchicalLogicalOperatorVisitor &visitor) { + if (visitor.PreVisit(*this)) { + left_op_->Accept(visitor) && right_op_->Accept(visitor); + } + return visitor.PostVisit(*this); +} + +WITHOUT_SINGLE_INPUT(Cartesian); + +namespace { + +class CartesianCursor : public Cursor { + public: + CartesianCursor(const Cartesian &self, utils::MemoryResource *mem) + : self_(self), + left_op_frames_(mem), + right_op_frame_(mem), + left_op_cursor_(self.left_op_->MakeCursor(mem)), + right_op_cursor_(self_.right_op_->MakeCursor(mem)) { + MG_ASSERT(left_op_cursor_ != nullptr, "CartesianCursor: Missing left operator cursor."); + MG_ASSERT(right_op_cursor_ != nullptr, "CartesianCursor: Missing right operator cursor."); + } + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("Cartesian"); + + if (!cartesian_pull_initialized_) { + // Pull all left_op frames. + while (left_op_cursor_->Pull(frame, context)) { + left_op_frames_.emplace_back(frame.elems().begin(), frame.elems().end()); + } + + // We're setting the iterator to 'end' here so it pulls the right + // cursor. + left_op_frames_it_ = left_op_frames_.end(); + cartesian_pull_initialized_ = true; + } + + // If left operator yielded zero results there is no cartesian product. + if (left_op_frames_.empty()) { + return false; + } + + auto restore_frame = [&frame](const auto &symbols, const auto &restore_from) { + for (const auto &symbol : symbols) { + frame[symbol] = restore_from[symbol.position()]; + } + }; + + if (left_op_frames_it_ == left_op_frames_.end()) { + // Advance right_op_cursor_. + if (!right_op_cursor_->Pull(frame, context)) return false; + + right_op_frame_.assign(frame.elems().begin(), frame.elems().end()); + left_op_frames_it_ = left_op_frames_.begin(); + } else { + // Make sure right_op_cursor last pulled results are on frame. + restore_frame(self_.right_symbols_, right_op_frame_); + } + + if (MustAbort(context)) throw HintedAbortError(); + + restore_frame(self_.left_symbols_, *left_op_frames_it_); + left_op_frames_it_++; + return true; + } + + void Shutdown() override { + left_op_cursor_->Shutdown(); + right_op_cursor_->Shutdown(); + } + + void Reset() override { + left_op_cursor_->Reset(); + right_op_cursor_->Reset(); + right_op_frame_.clear(); + left_op_frames_.clear(); + left_op_frames_it_ = left_op_frames_.end(); + cartesian_pull_initialized_ = false; + } + + private: + const Cartesian &self_; + utils::pmr::vector> left_op_frames_; + utils::pmr::vector right_op_frame_; + const UniqueCursorPtr left_op_cursor_; + const UniqueCursorPtr right_op_cursor_; + utils::pmr::vector>::iterator left_op_frames_it_; + bool cartesian_pull_initialized_{false}; +}; + +} // namespace + +UniqueCursorPtr Cartesian::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::CartesianOperator); + + return MakeUniqueCursorPtr(mem, *this, mem); +} + +OutputTable::OutputTable(std::vector output_symbols, std::vector> rows) + : output_symbols_(std::move(output_symbols)), callback_([rows](Frame *, ExecutionContext *) { return rows; }) {} + +OutputTable::OutputTable(std::vector output_symbols, + std::function>(Frame *, ExecutionContext *)> callback) + : output_symbols_(std::move(output_symbols)), callback_(std::move(callback)) {} + +WITHOUT_SINGLE_INPUT(OutputTable); + +class OutputTableCursor : public Cursor { + public: + OutputTableCursor(const OutputTable &self) : self_(self) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + if (!pulled_) { + rows_ = self_.callback_(&frame, &context); + for (const auto &row : rows_) { + MG_ASSERT(row.size() == self_.output_symbols_.size(), "Wrong number of columns in row!"); + } + pulled_ = true; + } + if (current_row_ < rows_.size()) { + for (size_t i = 0; i < self_.output_symbols_.size(); ++i) { + frame[self_.output_symbols_[i]] = rows_[current_row_][i]; + } + current_row_++; + return true; + } + return false; + } + + void Reset() override { + pulled_ = false; + current_row_ = 0; + rows_.clear(); + } + + void Shutdown() override {} + + private: + const OutputTable &self_; + size_t current_row_{0}; + std::vector> rows_; + bool pulled_{false}; +}; + +UniqueCursorPtr OutputTable::MakeCursor(utils::MemoryResource *mem) const { + return MakeUniqueCursorPtr(mem, *this); +} + +OutputTableStream::OutputTableStream( + std::vector output_symbols, + std::function>(Frame *, ExecutionContext *)> callback) + : output_symbols_(std::move(output_symbols)), callback_(std::move(callback)) {} + +WITHOUT_SINGLE_INPUT(OutputTableStream); + +class OutputTableStreamCursor : public Cursor { + public: + explicit OutputTableStreamCursor(const OutputTableStream *self) : self_(self) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + const auto row = self_->callback_(&frame, &context); + if (row) { + MG_ASSERT(row->size() == self_->output_symbols_.size(), "Wrong number of columns in row!"); + for (size_t i = 0; i < self_->output_symbols_.size(); ++i) { + frame[self_->output_symbols_[i]] = row->at(i); + } + return true; + } + return false; + } + + // TODO(tsabolcec): Come up with better approach for handling `Reset()`. + // One possibility is to implement a custom closure utility class with + // `Reset()` method. + void Reset() override { throw utils::NotYetImplemented("OutputTableStreamCursor::Reset"); } + + void Shutdown() override {} + + private: + const OutputTableStream *self_; +}; + +UniqueCursorPtr OutputTableStream::MakeCursor(utils::MemoryResource *mem) const { + return MakeUniqueCursorPtr(mem, this); +} + +CallProcedure::CallProcedure(std::shared_ptr input, std::string name, std::vector args, + std::vector fields, std::vector symbols, Expression *memory_limit, + size_t memory_scale, bool is_write) + : input_(input ? input : std::make_shared()), + procedure_name_(name), + arguments_(args), + result_fields_(fields), + result_symbols_(symbols), + memory_limit_(memory_limit), + memory_scale_(memory_scale), + is_write_(is_write) {} + +ACCEPT_WITH_INPUT(CallProcedure); + +std::vector CallProcedure::OutputSymbols(const SymbolTable &) const { return result_symbols_; } + +std::vector CallProcedure::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.insert(symbols.end(), result_symbols_.begin(), result_symbols_.end()); + return symbols; +} + +void CallProcedure::IncrementCounter(const std::string &procedure_name) { + procedure_counters_.WithLock([&](auto &counters) { ++counters[procedure_name]; }); +} + +std::unordered_map CallProcedure::GetAndResetCounters() { + auto counters = procedure_counters_.Lock(); + auto ret = std::move(*counters); + counters->clear(); + return ret; +} + +namespace { + +void CallCustomProcedure(const std::string_view fully_qualified_procedure_name, const mgp_proc &proc, + const std::vector &args, mgp_graph &graph, ExpressionEvaluator *evaluator, + utils::MemoryResource *memory, std::optional memory_limit, mgp_result *result) { + static_assert(std::uses_allocator_v>, + "Expected mgp_value to use custom allocator and makes STL " + "containers aware of that"); + // Build and type check procedure arguments. + mgp_list proc_args(memory); + std::vector args_list; + args_list.reserve(args.size()); + for (auto *expression : args) { + args_list.emplace_back(expression->Accept(*evaluator)); + } + procedure::ConstructArguments(args_list, proc, fully_qualified_procedure_name, proc_args, graph); + if (memory_limit) { + SPDLOG_INFO("Running '{}' with memory limit of {}", fully_qualified_procedure_name, + utils::GetReadableSize(*memory_limit)); + utils::LimitedMemoryResource limited_mem(memory, *memory_limit); + mgp_memory proc_memory{&limited_mem}; + MG_ASSERT(result->signature == &proc.results); + // TODO: What about cross library boundary exceptions? OMG C++?! + proc.cb(&proc_args, &graph, result, &proc_memory); + size_t leaked_bytes = limited_mem.GetAllocatedBytes(); + if (leaked_bytes > 0U) { + spdlog::warn("Query procedure '{}' leaked {} *tracked* bytes", fully_qualified_procedure_name, leaked_bytes); + } + } else { + // TODO: Add a tracking MemoryResource without limits, so that we report + // memory leaks in procedure. + mgp_memory proc_memory{memory}; + MG_ASSERT(result->signature == &proc.results); + // TODO: What about cross library boundary exceptions? OMG C++?! + proc.cb(&proc_args, &graph, result, &proc_memory); + } +} + +} // namespace + +class CallProcedureCursor : public Cursor { + const CallProcedure *self_; + UniqueCursorPtr input_cursor_; + mgp_result result_; + decltype(result_.rows.end()) result_row_it_{result_.rows.end()}; + size_t result_signature_size_{0}; + + public: + CallProcedureCursor(const CallProcedure *self, utils::MemoryResource *mem) + : self_(self), + input_cursor_(self_->input_->MakeCursor(mem)), + // result_ needs to live throughout multiple Pull evaluations, until all + // rows are produced. Therefore, we use the memory dedicated for the + // whole execution. + result_(nullptr, mem) { + MG_ASSERT(self_->result_fields_.size() == self_->result_symbols_.size(), "Incorrectly constructed CallProcedure"); + } + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("CallProcedure"); + + if (MustAbort(context)) throw HintedAbortError(); + + // We need to fetch new procedure results after pulling from input. + // TODO: Look into openCypher's distinction between procedures returning an + // empty result set vs procedures which return `void`. We currently don't + // have procedures registering what they return. + // This `while` loop will skip over empty results. + while (result_row_it_ == result_.rows.end()) { + if (!input_cursor_->Pull(frame, context)) return false; + result_.signature = nullptr; + result_.rows.clear(); + result_.error_msg.reset(); + // It might be a good idea to resolve the procedure name once, at the + // start. Unfortunately, this could deadlock if we tried to invoke a + // procedure from a module (read lock) and reload a module (write lock) + // inside the same execution thread. Also, our RWLock is setup so that + // it's not possible for a single thread to request multiple read locks. + // Builtin module registration in query/procedure/module.cpp depends on + // this locking scheme. + const auto &maybe_found = procedure::FindProcedure(procedure::gModuleRegistry, self_->procedure_name_, + context.evaluation_context.memory); + if (!maybe_found) { + throw QueryRuntimeException("There is no procedure named '{}'.", self_->procedure_name_); + } + const auto &[module, proc] = *maybe_found; + if (proc->info.is_write != self_->is_write_) { + auto get_proc_type_str = [](bool is_write) { return is_write ? "write" : "read"; }; + throw QueryRuntimeException("The procedure named '{}' was a {} procedure, but changed to be a {} procedure.", + self_->procedure_name_, get_proc_type_str(self_->is_write_), + get_proc_type_str(proc->info.is_write)); + } + const auto graph_view = proc->info.is_write ? storage::v3::View::NEW : storage::v3::View::OLD; + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + graph_view); + + result_.signature = &proc->results; + // Use evaluation memory, as invoking a procedure is akin to a simple + // evaluation of an expression. + // TODO: This will probably need to be changed when we add support for + // generator like procedures which yield a new result on each invocation. + auto *memory = context.evaluation_context.memory; + auto memory_limit = EvaluateMemoryLimit(&evaluator, self_->memory_limit_, self_->memory_scale_); + auto graph = mgp_graph::WritableGraph(*context.db_accessor, graph_view, context); + CallCustomProcedure(self_->procedure_name_, *proc, self_->arguments_, graph, &evaluator, memory, memory_limit, + &result_); + + // Reset result_.signature to nullptr, because outside of this scope we + // will no longer hold a lock on the `module`. If someone were to reload + // it, the pointer would be invalid. + result_signature_size_ = result_.signature->size(); + result_.signature = nullptr; + if (result_.error_msg) { + throw QueryRuntimeException("{}: {}", self_->procedure_name_, *result_.error_msg); + } + result_row_it_ = result_.rows.begin(); + } + + const auto &values = result_row_it_->values; + // Check that the row has all fields as required by the result signature. + // C API guarantees that it's impossible to set fields which are not part of + // the result record, but it does not gurantee that some may be missing. See + // `mgp_result_record_insert`. + if (values.size() != result_signature_size_) { + throw QueryRuntimeException( + "Procedure '{}' did not yield all fields as required by its " + "signature.", + self_->procedure_name_); + } + for (size_t i = 0; i < self_->result_fields_.size(); ++i) { + std::string_view field_name(self_->result_fields_[i]); + auto result_it = values.find(field_name); + if (result_it == values.end()) { + throw QueryRuntimeException("Procedure '{}' did not yield a record with '{}' field.", self_->procedure_name_, + field_name); + } + frame[self_->result_symbols_[i]] = result_it->second; + } + ++result_row_it_; + + return true; + } + + void Reset() override { + result_.rows.clear(); + result_.error_msg.reset(); + input_cursor_->Reset(); + } + + void Shutdown() override {} +}; + +UniqueCursorPtr CallProcedure::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::CallProcedureOperator); + CallProcedure::IncrementCounter(procedure_name_); + + return MakeUniqueCursorPtr(mem, this, mem); +} + +LoadCsv::LoadCsv(std::shared_ptr input, Expression *file, bool with_header, bool ignore_bad, + Expression *delimiter, Expression *quote, Symbol row_var) + : input_(input ? input : (std::make_shared())), + file_(file), + with_header_(with_header), + ignore_bad_(ignore_bad), + delimiter_(delimiter), + quote_(quote), + row_var_(row_var) { + MG_ASSERT(file_, "Something went wrong - '{}' member file_ shouldn't be a nullptr", __func__); +} + +bool LoadCsv::Accept(HierarchicalLogicalOperatorVisitor &visitor) { return false; }; + +class LoadCsvCursor; + +std::vector LoadCsv::OutputSymbols(const SymbolTable &sym_table) const { return {row_var_}; }; + +std::vector LoadCsv::ModifiedSymbols(const SymbolTable &sym_table) const { + auto symbols = input_->ModifiedSymbols(sym_table); + symbols.push_back(row_var_); + return symbols; +}; + +namespace { +// copy-pasted from interpreter.cpp +TypedValue EvaluateOptionalExpression(Expression *expression, ExpressionEvaluator *eval) { + return expression ? expression->Accept(*eval) : TypedValue(); +} + +auto ToOptionalString(ExpressionEvaluator *evaluator, Expression *expression) -> std::optional { + const auto evaluated_expr = EvaluateOptionalExpression(expression, evaluator); + if (evaluated_expr.IsString()) { + return utils::pmr::string(evaluated_expr.ValueString(), utils::NewDeleteResource()); + } + return std::nullopt; +}; + +TypedValue CsvRowToTypedList(csv::Reader::Row row) { + auto *mem = row.get_allocator().GetMemoryResource(); + auto typed_columns = utils::pmr::vector(mem); + typed_columns.reserve(row.size()); + for (auto &column : row) { + typed_columns.emplace_back(std::move(column)); + } + return TypedValue(typed_columns, mem); +} + +TypedValue CsvRowToTypedMap(csv::Reader::Row row, csv::Reader::Header header) { + // a valid row has the same number of elements as the header + auto *mem = row.get_allocator().GetMemoryResource(); + utils::pmr::map m(mem); + for (auto i = 0; i < row.size(); ++i) { + m.emplace(std::move(header[i]), std::move(row[i])); + } + return TypedValue(m, mem); +} + +} // namespace + +class LoadCsvCursor : public Cursor { + const LoadCsv *self_; + const UniqueCursorPtr input_cursor_; + bool input_is_once_; + std::optional reader_{}; + + public: + LoadCsvCursor(const LoadCsv *self, utils::MemoryResource *mem) + : self_(self), input_cursor_(self_->input_->MakeCursor(mem)) { + input_is_once_ = dynamic_cast(self_->input_.get()); + } + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP("LoadCsv"); + + if (MustAbort(context)) throw HintedAbortError(); + + // ToDo(the-joksim): + // - this is an ungodly hack because the pipeline of creating a plan + // doesn't allow evaluating the expressions contained in self_->file_, + // self_->delimiter_, and self_->quote_ earlier (say, in the interpreter.cpp) + // without massacring the code even worse than I did here + if (UNLIKELY(!reader_)) { + reader_ = MakeReader(&context.evaluation_context); + } + + bool input_pulled = input_cursor_->Pull(frame, context); + + // If the input is Once, we have to keep going until we read all the rows, + // regardless of whether the pull on Once returned false. + // If we have e.g. MATCH(n) LOAD CSV ... AS x SET n.name = x.name, then we + // have to read at most cardinality(n) rows (but we can read less and stop + // pulling MATCH). + if (!input_is_once_ && !input_pulled) return false; + + if (auto row = reader_->GetNextRow(context.evaluation_context.memory)) { + if (!reader_->HasHeader()) { + frame[self_->row_var_] = CsvRowToTypedList(std::move(*row)); + } else { + frame[self_->row_var_] = CsvRowToTypedMap( + std::move(*row), csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory)); + } + return true; + } + + return false; + } + + void Reset() override { input_cursor_->Reset(); } + void Shutdown() override { input_cursor_->Shutdown(); } + + private: + csv::Reader MakeReader(EvaluationContext *eval_context) { + Frame frame(0); + SymbolTable symbol_table; + DbAccessor *dba = nullptr; + auto evaluator = ExpressionEvaluator(&frame, symbol_table, *eval_context, dba, storage::v3::View::OLD); + + auto maybe_file = ToOptionalString(&evaluator, self_->file_); + auto maybe_delim = ToOptionalString(&evaluator, self_->delimiter_); + auto maybe_quote = ToOptionalString(&evaluator, self_->quote_); + + // No need to check if maybe_file is std::nullopt, as the parser makes sure + // we can't get a nullptr for the 'file_' member in the LoadCsv clause. + // Note that the reader has to be given its own memory resource, as it + // persists between pulls, so it can't use the evalutation context memory + // resource. + return csv::Reader( + *maybe_file, + csv::Reader::Config(self_->with_header_, self_->ignore_bad_, std::move(maybe_delim), std::move(maybe_quote)), + utils::NewDeleteResource()); + } +}; + +UniqueCursorPtr LoadCsv::MakeCursor(utils::MemoryResource *mem) const { + return MakeUniqueCursorPtr(mem, this, mem); +}; + +class ForeachCursor : public Cursor { + public: + explicit ForeachCursor(const Foreach &foreach, utils::MemoryResource *mem) + : loop_variable_symbol_(foreach.loop_variable_symbol_), + input_(foreach.input_->MakeCursor(mem)), + updates_(foreach.update_clauses_->MakeCursor(mem)), + expression(foreach.expression_) {} + + bool Pull(Frame &frame, ExecutionContext &context) override { + SCOPED_PROFILE_OP(op_name_); + + if (!input_->Pull(frame, context)) { + return false; + } + + ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor, + storage::v3::View::NEW); + TypedValue expr_result = expression->Accept(evaluator); + + if (expr_result.IsNull()) { + return true; + } + + if (!expr_result.IsList()) { + throw QueryRuntimeException("FOREACH expression must resolve to a list, but got '{}'.", expr_result.type()); + } + + const auto &cache_ = expr_result.ValueList(); + for (const auto &index : cache_) { + frame[loop_variable_symbol_] = index; + while (updates_->Pull(frame, context)) { + } + ResetUpdates(); + } + + return true; + } + + void Shutdown() override { input_->Shutdown(); } + + void ResetUpdates() { updates_->Reset(); } + + void Reset() override { + input_->Reset(); + ResetUpdates(); + } + + private: + const Symbol loop_variable_symbol_; + const UniqueCursorPtr input_; + const UniqueCursorPtr updates_; + Expression *expression; + const char *op_name_{"Foreach"}; +}; + +Foreach::Foreach(std::shared_ptr input, std::shared_ptr updates, Expression *expr, + Symbol loop_variable_symbol) + : input_(input ? std::move(input) : std::make_shared()), + update_clauses_(std::move(updates)), + expression_(expr), + loop_variable_symbol_(loop_variable_symbol) {} + +UniqueCursorPtr Foreach::MakeCursor(utils::MemoryResource *mem) const { + EventCounter::IncrementCounter(EventCounter::ForeachOperator); + return MakeUniqueCursorPtr(mem, *this, mem); +} + +std::vector Foreach::ModifiedSymbols(const SymbolTable &table) const { + auto symbols = input_->ModifiedSymbols(table); + symbols.emplace_back(loop_variable_symbol_); + return symbols; +} + +bool Foreach::Accept(HierarchicalLogicalOperatorVisitor &visitor) { + if (visitor.PreVisit(*this)) { + input_->Accept(visitor); + update_clauses_->Accept(visitor); + } + return visitor.PostVisit(*this); +} + +} // namespace memgraph::query::v2::plan diff --git a/src/query/v2/plan/operator.lcp b/src/query/v2/plan/operator.lcp new file mode 100644 index 000000000..393529b77 --- /dev/null +++ b/src/query/v2/plan/operator.lcp @@ -0,0 +1,2305 @@ +;; Copyright 2022 Memgraph Ltd. +;; +;; Use of this software is governed by the Business Source License +;; included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +;; License, and you may not use this file except in compliance with the Business Source License. +;; +;; As of the Change Date specified in that file, in accordance with +;; the Business Source License, use of this software will be governed +;; by the Apache License, Version 2.0, included in the file +;; licenses/APL.txt. + +#>cpp +/** @file */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "query/v2/common.hpp" +#include "query/v2/frontend/ast/ast.hpp" +#include "query/v2/frontend/semantic/symbol.hpp" +#include "query/v2/typed_value.hpp" +#include "storage/v3/id_types.hpp" +#include "utils/bound.hpp" +#include "utils/fnv.hpp" +#include "utils/memory.hpp" +#include "utils/visitor.hpp" +#include "utils/logging.hpp" +cpp<# + +(lcp:namespace memgraph) +(lcp:namespace query) +(lcp:namespace v2) + +#>cpp +struct ExecutionContext; +class ExpressionEvaluator; +class Frame; +class SymbolTable; +cpp<# + +(lcp:namespace plan) + +#>cpp +/// Base class for iteration cursors of @c LogicalOperator classes. +/// +/// Each @c LogicalOperator must produce a concrete @c Cursor, which provides +/// the iteration mechanism. +class Cursor { + public: + /// Run an iteration of a @c LogicalOperator. + /// + /// Since operators may be chained, the iteration may pull results from + /// multiple operators. + /// + /// @param Frame May be read from or written to while performing the + /// iteration. + /// @param ExecutionContext Used to get the position of symbols in frame and + /// other information. + /// + /// @throws QueryRuntimeException if something went wrong with execution + virtual bool Pull(Frame &, ExecutionContext &) = 0; + + /// Resets the Cursor to its initial state. + virtual void Reset() = 0; + + /// Perform cleanup which may throw an exception + virtual void Shutdown() = 0; + + virtual ~Cursor() {} +}; + +/// unique_ptr to Cursor managed with a custom deleter. +/// This allows us to use utils::MemoryResource for allocation. +using UniqueCursorPtr = std::unique_ptr>; + +template +std::unique_ptr> MakeUniqueCursorPtr( + utils::Allocator allocator, TArgs &&... args) { + auto *ptr = allocator.allocate(1); + try { + auto *cursor = new (ptr) TCursor(std::forward(args)...); + return std::unique_ptr>( + cursor, [allocator](Cursor *base_ptr) mutable { + auto *p = static_cast(base_ptr); + p->~TCursor(); + allocator.deallocate(p, 1); + }); + } catch (...) { + allocator.deallocate(ptr, 1); + throw; + } +} + +class Once; +class CreateNode; +class CreateExpand; +class ScanAll; +class ScanAllByLabel; +class ScanAllByLabelPropertyRange; +class ScanAllByLabelPropertyValue; +class ScanAllByLabelProperty; +class ScanAllById; +class Expand; +class ExpandVariable; +class ConstructNamedPath; +class Filter; +class Produce; +class Delete; +class SetProperty; +class SetProperties; +class SetLabels; +class RemoveProperty; +class RemoveLabels; +class EdgeUniquenessFilter; +class Accumulate; +class Aggregate; +class Skip; +class Limit; +class OrderBy; +class Merge; +class Optional; +class Unwind; +class Distinct; +class Union; +class Cartesian; +class CallProcedure; +class LoadCsv; +class Foreach; + +using LogicalOperatorCompositeVisitor = utils::CompositeVisitor< + Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel, + ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue, + ScanAllByLabelProperty, ScanAllById, + Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete, + SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels, + EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge, + Optional, Unwind, Distinct, Union, Cartesian, CallProcedure, LoadCsv, Foreach>; + +using LogicalOperatorLeafVisitor = utils::LeafVisitor; + +/** + * @brief Base class for hierarhical visitors of @c LogicalOperator class + * hierarchy. + */ +class HierarchicalLogicalOperatorVisitor + : public LogicalOperatorCompositeVisitor, + public LogicalOperatorLeafVisitor { + public: + using LogicalOperatorCompositeVisitor::PostVisit; + using LogicalOperatorCompositeVisitor::PreVisit; + using LogicalOperatorLeafVisitor::Visit; + using typename LogicalOperatorLeafVisitor::ReturnType; +}; +cpp<# + +(lcp:define-class logical-operator ("utils::Visitable") + () + (:abstractp t) + (:documentation + "Base class for logical operators. + +Each operator describes an operation, which is to be performed on the +database. Operators are iterated over using a @c Cursor. Various operators +can serve as inputs to others and thus a sequence of operations is formed.") + (:public + #>cpp + virtual ~LogicalOperator() {} + + /** Construct a @c Cursor which is used to run this operator. + * + * @param utils::MemoryResource Memory resource used for allocations during + * the lifetime of the returned Cursor. + */ + virtual UniqueCursorPtr MakeCursor(utils::MemoryResource *) const = 0; + + /** Return @c Symbol vector where the query results will be stored. + * + * Currently, output symbols are generated in @c Produce @c Union and + * @c CallProcedure operators. @c Skip, @c Limit, @c OrderBy and @c Distinct + * propagate the symbols from @c Produce (if it exists as input operator). + * + * @param SymbolTable used to find symbols for expressions. + * @return std::vector used for results. + */ + virtual std::vector OutputSymbols(const SymbolTable &) const { + return std::vector(); + } + + /** + * Symbol vector whose values are modified by this operator sub-tree. + * + * This is different than @c OutputSymbols, because it returns all of the + * modified symbols, including those that may not be returned as the + * result of the query. Note that the modified symbols will not contain + * those that should not be read after the operator is processed. + * + * For example, `MATCH (n)-[e]-(m) RETURN n AS l` will generate `ScanAll (n) > + * Expand (e, m) > Produce (l)`. The modified symbols on Produce sub-tree will + * be `l`, the same as output symbols, because it isn't valid to read `n`, `e` + * nor `m` after Produce. On the other hand, modified symbols from Expand + * contain `e` and `m`, as well as `n`, while output symbols are empty. + * Modified symbols from ScanAll contain only `n`, while output symbols are + * also empty. + */ + virtual std::vector ModifiedSymbols(const SymbolTable &) const = 0; + + /** + * Returns true if the operator takes only one input operator. + * NOTE: When this method returns true, you may use `input` and `set_input` + * methods. + */ + virtual bool HasSingleInput() const = 0; + + /** + * Returns the input operator if it has any. + * NOTE: This should only be called if `HasSingleInput() == true`. + */ + virtual std::shared_ptr input() const = 0; + /** + * Set a different input on this operator. + * NOTE: This should only be called if `HasSingleInput() == true`. + */ + virtual void set_input(std::shared_ptr) = 0; + + struct SaveHelper { + std::vector saved_ops; + }; + + struct LoadHelper { + AstStorage ast_storage; + std::vector>> + loaded_ops; + }; + + struct SlkLoadHelper { + AstStorage ast_storage; + std::vector> loaded_ops; + }; + cpp<#) + (:serialize + (:slk :base t + :save-args '((helper "query::v2::plan::LogicalOperator::SaveHelper *")) + :load-args '((helper "query::v2::plan::LogicalOperator::SlkLoadHelper *")))) + (:type-info :base t) + (:clone :args '((storage "AstStorage *")) + :base t)) + +(defun slk-save-ast-pointer (member) + #>cpp + query::v2::SaveAstPointer(self.${member}, builder); + cpp<#) + +(defun slk-load-ast-pointer (type) + (lambda (member) + #>cpp + self->${member} = query::v2::LoadAstPointer( + &helper->ast_storage, reader); + cpp<#)) + +(defun slk-save-ast-vector (member) + #>cpp + size_t size = self.${member}.size(); + slk::Save(size, builder); + for (const auto *val : self.${member}) { + query::v2::SaveAstPointer(val, builder); + } + cpp<#) + +(defun slk-load-ast-vector (type) + (lambda (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; + i < size; + ++i) { + self->${member}[i] = query::v2::LoadAstPointer( + &helper->ast_storage, reader); + } + cpp<#)) + +(defun slk-save-operator-pointer (member) + #>cpp + slk::Save(self.${member}, builder, + &helper->saved_ops, + [&helper](const auto &val, + auto *builder) { + slk::Save(val, builder, helper); + }); + cpp<#) + +(defun slk-load-operator-pointer (member) + #>cpp + slk::Load(&self->${member}, reader, &helper->loaded_ops, + [&helper](auto *op, auto *reader) { + slk::ConstructAndLoad(op, reader, helper); + }); + cpp<#) + +(lcp:define-class once (logical-operator) + ((symbols "std::vector" :scope :public)) + (:documentation + "A logical operator whose Cursor returns true on the first Pull +and false on every following Pull.") + (:public + #>cpp + Once(std::vector symbols = {}) : symbols_{std::move(symbols)} {} + DEFVISITABLE(HierarchicalLogicalOperatorVisitor); + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override { + return symbols_; + } + + bool HasSingleInput() const override; + std::shared_ptr input() const override; + void set_input(std::shared_ptr) override; + cpp<#) + (:private + #>cpp + class OnceCursor : public Cursor { + public: + OnceCursor() {} + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + bool did_pull_{false}; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(defun slk-save-properties (member) + #>cpp + size_t size = self.${member}.size(); + slk::Save(size, builder); + for (const auto &kv : self.${member}) { + slk::Save(kv.first, builder); + query::v2::SaveAstPointer(kv.second, builder); + } + cpp<#) + +(defun slk-load-properties (member) + #>cpp + size_t size = 0; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; i < size; ++i) { + storage::v3::PropertyId prop; + slk::Load(&prop, reader); + auto *expr = query::v2::LoadAstPointer( + &helper->ast_storage, reader); + self->${member}[i] = {prop, expr}; + } + cpp<#) + +(defun clone-variant-properties (source destination) + #>cpp + if (const auto *props = std::get_if(&${source})) { + auto &destination_props = std::get(${destination}); + destination_props.resize(props->size()); + for (auto i0 = 0; i0 < props->size(); ++i0) { + { + storage::v3::PropertyId first1 = (*props)[i0].first; + Expression *second2; + second2 = (*props)[i0].second ? (*props)[i0].second->Clone(storage) : nullptr; + destination_props[i0] = std::make_pair(std::move(first1), std::move(second2)); + } + } + } else { + ${destination} = std::get(${source})->Clone(storage); + } + cpp<#) + +#>cpp +using PropertiesMapList = std::vector>; +cpp<# + +(lcp:define-struct node-creation-info () + ((symbol "Symbol") + (labels "std::vector") + (properties "std::variant" + :slk-save #'slk-save-properties + :slk-load #'slk-load-properties + :clone #'clone-variant-properties)) + (:serialize (:slk :save-args '((helper "query::v2::plan::LogicalOperator::SaveHelper *")) + :load-args '((helper "query::v2::plan::LogicalOperator::SlkLoadHelper *")))) + (:clone :args '((storage "AstStorage *"))) + (:public + #>cpp + NodeCreationInfo() = default; + + NodeCreationInfo( + Symbol symbol, std::vector labels, + std::variant properties) + : symbol{std::move(symbol)}, labels{std::move(labels)}, properties{std::move(properties)} {}; + + NodeCreationInfo(Symbol symbol, std::vector labels, + PropertiesMapList properties) + : symbol{std::move(symbol)}, labels{std::move(labels)}, properties{std::move(properties)} {}; + + NodeCreationInfo(Symbol symbol, std::vector labels, ParameterLookup* properties) + : symbol{std::move(symbol)}, labels{std::move(labels)}, properties{properties} {}; + cpp<#)) + +(lcp:define-class create-node (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (node-info "NodeCreationInfo" :scope :public + :slk-save (lambda (m) + #>cpp + slk::Save(self.${m}, builder, helper); + cpp<#) + :slk-load (lambda (m) + #>cpp + slk::Load(&self->${m}, reader, helper); + cpp<#))) + (:documentation + "Operator for creating a node. + +This op is used both for creating a single node (`CREATE` statement without +a preceding `MATCH`), or multiple nodes (`MATCH ... CREATE` or +`CREATE (), () ...`). + +@sa CreateExpand") + (:public + #>cpp + CreateNode() {} + + /** + * @param input Optional. If @c nullptr, then a single node will be + * created (a single successful @c Cursor::Pull from this op's @c Cursor). + * If a valid input, then a node will be created for each + * successful pull from the given input. + * @param node_info @c NodeCreationInfo + */ + CreateNode(const std::shared_ptr &input, + const NodeCreationInfo &node_info); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class CreateNodeCursor : public Cursor { + public: + CreateNodeCursor(const CreateNode &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const CreateNode &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-struct edge-creation-info () + ((symbol "Symbol") + (properties "std::variant" + :slk-save #'slk-save-properties + :slk-load #'slk-load-properties + :clone #'clone-variant-properties) + (edge-type "::storage::v3::EdgeTypeId") + (direction "::EdgeAtom::Direction" :initval "EdgeAtom::Direction::BOTH")) + (:serialize (:slk :save-args '((helper "query::v2::plan::LogicalOperator::SaveHelper *")) + :load-args '((helper "query::v2::plan::LogicalOperator::SlkLoadHelper *")))) + (:clone :args '((storage "AstStorage *"))) + (:public + #>cpp + EdgeCreationInfo() = default; + + EdgeCreationInfo(Symbol symbol, std::variant properties, + storage::v3::EdgeTypeId edge_type, EdgeAtom::Direction direction) + : symbol{std::move(symbol)}, properties{std::move(properties)}, edge_type{edge_type}, direction{direction} {}; + + EdgeCreationInfo(Symbol symbol, PropertiesMapList properties, + storage::v3::EdgeTypeId edge_type, EdgeAtom::Direction direction) + : symbol{std::move(symbol)}, properties{std::move(properties)}, edge_type{edge_type}, direction{direction} {}; + + EdgeCreationInfo(Symbol symbol, ParameterLookup* properties, + storage::v3::EdgeTypeId edge_type, EdgeAtom::Direction direction) + : symbol{std::move(symbol)}, properties{properties}, edge_type{edge_type}, direction{direction} {}; + cpp<#)) + +(lcp:define-class create-expand (logical-operator) + ((node-info "NodeCreationInfo" :scope :public + :slk-save (lambda (m) + #>cpp + slk::Save(self.${m}, builder, helper); + cpp<#) + :slk-load (lambda (m) + #>cpp + slk::Load(&self->${m}, reader, helper); + cpp<#)) + (edge-info "EdgeCreationInfo" :scope :public + :slk-save (lambda (m) + #>cpp + slk::Save(self.${m}, builder, helper); + cpp<#) + :slk-load (lambda (m) + #>cpp + slk::Load(&self->${m}, reader, helper); + cpp<#)) + ;; the input op and the symbol under which the op's result + ;; can be found in the frame + (input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (input-symbol "Symbol" :scope :public) + (existing-node :bool :scope :public :documentation + "if the given node atom refers to an existing node (either matched or created)")) + (:documentation + "Operator for creating edges and destination nodes. + +This operator extends already created nodes with an edge. If the other node +on the edge does not exist, it will be created. For example, in `MATCH (n) +CREATE (n) -[r:r]-> (n)` query, this operator will create just the edge `r`. +In `MATCH (n) CREATE (n) -[r:r]-> (m)` query, the operator will create both +the edge `r` and the node `m`. In case of `CREATE (n) -[r:r]-> (m)` the +first node `n` is created by @c CreateNode operator, while @c CreateExpand +will create the edge `r` and `m`. Similarly, multiple @c CreateExpand are +chained in cases when longer paths need creating. + +@sa CreateNode") + (:public + #>cpp + CreateExpand() {} + + /** @brief Construct @c CreateExpand. + * + * @param node_info @c NodeCreationInfo at the end of the edge. + * Used to create a node, unless it refers to an existing one. + * @param edge_info @c EdgeCreationInfo for the edge to be created. + * @param input Optional. Previous @c LogicalOperator which will be pulled. + * For each successful @c Cursor::Pull, this operator will create an + * expansion. + * @param input_symbol @c Symbol for the node at the start of the edge. + * @param existing_node @c bool indicating whether the @c node_atom refers to + * an existing node. If @c false, the operator will also create the node. + */ + CreateExpand(const NodeCreationInfo &node_info, + const EdgeCreationInfo &edge_info, + const std::shared_ptr &input, + Symbol input_symbol, bool existing_node); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class CreateExpandCursor : public Cursor { + public: + CreateExpandCursor(const CreateExpand &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const CreateExpand &self_; + const UniqueCursorPtr input_cursor_; + + // Get the existing node (if existing_node_ == true), or create a new node + VertexAccessor &OtherVertex(Frame &frame, ExecutionContext &context); + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class scan-all (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (output-symbol "Symbol" :scope :public) + (view "::storage::v3::View" :scope :public + :documentation + "Controls which graph state is used to produce vertices. + +If @c storage::v3::View::OLD, @c ScanAll will produce vertices visible in the +previous graph state, before modifications done by current transaction & +command. With @c storage::v3::View::NEW, all vertices will be produced the current +transaction sees along with their modifications.")) + + (:documentation + "Operator which iterates over all the nodes currently in the database. +When given an input (optional), does a cartesian product. + +It accepts an optional input. If provided then this op scans all the nodes +currently in the database for each successful Pull from it's input, thereby +producing a cartesian product of input Pulls and database elements. + +ScanAll can either iterate over the previous graph state (state before +the current transacton+command) or over current state. This is controlled +with a constructor argument. + +@sa ScanAllByLabel +@sa ScanAllByLabelPropertyRange +@sa ScanAllByLabelPropertyValue") + (:public + #>cpp + ScanAll() {} + ScanAll(const std::shared_ptr &input, Symbol output_symbol, + storage::v3::View view = storage::v3::View::OLD); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class scan-all-by-label (scan-all) + ((label "::storage::v3::LabelId" :scope :public)) + (:documentation + "Behaves like @c ScanAll, but this operator produces only vertices with +given label. + +@sa ScanAll +@sa ScanAllByLabelPropertyRange +@sa ScanAllByLabelPropertyValue") + (:public + #>cpp + ScanAllByLabel() {} + ScanAllByLabel(const std::shared_ptr &input, + Symbol output_symbol, storage::v3::LabelId label, + storage::v3::View view = storage::v3::View::OLD); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(defun slk-save-optional-bound (member) + #>cpp + slk::Save(static_cast(self.${member}), builder); + if (!self.${member}) { + return; + } + uint8_t bound_type; + const auto &bound = *self.${member}; + switch (bound.type()) { + case utils::BoundType::INCLUSIVE: + bound_type = 0; + break; + case utils::BoundType::EXCLUSIVE: + bound_type = 1; + break; + } + slk::Save(bound_type, builder); + query::v2::SaveAstPointer(bound.value(), builder); + cpp<#) + +(defun slk-load-optional-bound (member) + #>cpp + bool has_bound; + slk::Load(&has_bound, reader); + if (!has_bound) { + self->${member} = std::nullopt; + return; + } + uint8_t bound_type_value; + slk::Load(&bound_type_value, reader); + utils::BoundType bound_type; + switch (bound_type_value) { + case static_cast(0): + bound_type = utils::BoundType::INCLUSIVE; + break; + case static_cast(1): + bound_type = utils::BoundType::EXCLUSIVE; + break; + default: + throw slk::SlkDecodeException("Loading unknown BoundType"); + } + auto *value = query::v2::LoadAstPointer( + &helper->ast_storage, reader); + self->${member}.emplace(utils::Bound(value, bound_type)); + cpp<#) + +(defun clone-optional-bound (source dest) + #>cpp + if (${source}) { + ${dest}.emplace(utils::Bound( + ${source}->value()->Clone(storage), + ${source}->type())); + } else { + ${dest} = std::nullopt; + } + cpp<#) + +(lcp:define-class scan-all-by-label-property-range (scan-all) + ((label "::storage::v3::LabelId" :scope :public) + (property "::storage::v3::PropertyId" :scope :public) + (property-name "std::string" :scope :public) + (lower-bound "std::optional" :scope :public + :slk-save #'slk-save-optional-bound + :slk-load #'slk-load-optional-bound + :clone #'clone-optional-bound) + (upper-bound "std::optional" :scope :public + :slk-save #'slk-save-optional-bound + :slk-load #'slk-load-optional-bound + :clone #'clone-optional-bound)) + (:documentation + "Behaves like @c ScanAll, but produces only vertices with given label and +property value which is inside a range (inclusive or exlusive). + +@sa ScanAll +@sa ScanAllByLabel +@sa ScanAllByLabelPropertyValue") + (:public + #>cpp + /** Bound with expression which when evaluated produces the bound value. */ + using Bound = utils::Bound; + ScanAllByLabelPropertyRange() {} + /** + * Constructs the operator for given label and property value in range + * (inclusive). + * + * Range bounds are optional, but only one bound can be left out. + * + * @param input Preceding operator which will serve as the input. + * @param output_symbol Symbol where the vertices will be stored. + * @param label Label which the vertex must have. + * @param property Property from which the value will be looked up from. + * @param lower_bound Optional lower @c Bound. + * @param upper_bound Optional upper @c Bound. + * @param view storage::v3::View used when obtaining vertices. + */ + ScanAllByLabelPropertyRange(const std::shared_ptr &input, + Symbol output_symbol, storage::v3::LabelId label, + storage::v3::PropertyId property, + const std::string &property_name, + std::optional lower_bound, + std::optional upper_bound, + storage::v3::View view = storage::v3::View::OLD); + + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class scan-all-by-label-property-value (scan-all) + ((label "::storage::v3::LabelId" :scope :public) + (property "::storage::v3::PropertyId" :scope :public) + (property-name "std::string" :scope :public) + (expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:documentation + "Behaves like @c ScanAll, but produces only vertices with given label and +property value. + +@sa ScanAll +@sa ScanAllByLabel +@sa ScanAllByLabelPropertyRange") + (:public + #>cpp + ScanAllByLabelPropertyValue() {} + /** + * Constructs the operator for given label and property value. + * + * @param input Preceding operator which will serve as the input. + * @param output_symbol Symbol where the vertices will be stored. + * @param label Label which the vertex must have. + * @param property Property from which the value will be looked up from. + * @param expression Expression producing the value of the vertex property. + * @param view storage::v3::View used when obtaining vertices. + */ + ScanAllByLabelPropertyValue(const std::shared_ptr &input, + Symbol output_symbol, storage::v3::LabelId label, + storage::v3::PropertyId property, + const std::string &property_name, + Expression *expression, + storage::v3::View view = storage::v3::View::OLD); + + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class scan-all-by-label-property (scan-all) + ((label "::storage::v3::LabelId" :scope :public) + (property "::storage::v3::PropertyId" :scope :public) + (property-name "std::string" :scope :public) + (expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + + (:documentation + "Behaves like @c ScanAll, but this operator produces only vertices with +given label and property. + +@sa ScanAll +@sa ScanAllByLabelPropertyRange +@sa ScanAllByLabelPropertyValue") + (:public + #>cpp + ScanAllByLabelProperty() {} + ScanAllByLabelProperty(const std::shared_ptr &input, + Symbol output_symbol, storage::v3::LabelId label, + storage::v3::PropertyId property, + const std::string &property_name, + storage::v3::View view = storage::v3::View::OLD); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + cpp<#) + (:serialize (:slk)) + (:clone)) + + + +(lcp:define-class scan-all-by-id (scan-all) + ((expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:documentation + "ScanAll producing a single node with ID equal to evaluated expression") + (:public + #>cpp + ScanAllById() {} + ScanAllById(const std::shared_ptr &input, + Symbol output_symbol, Expression *expression, + storage::v3::View view = storage::v3::View::OLD); + + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-struct expand-common () + ( + ;; info on what's getting expanded + (node-symbol "Symbol" + :documentation "Symbol pointing to the node to be expanded. +This is where the new node will be stored.") + (edge-symbol "Symbol" + :documentation "Symbol for the edges to be expanded. +This is where a TypedValue containing a list of expanded edges will be stored.") + (direction "::EdgeAtom::Direction" + :documentation "EdgeAtom::Direction determining the direction of edge +expansion. The direction is relative to the starting vertex for each expansion.") + (edge-types "std::vector" + :documentation "storage::v3::EdgeTypeId specifying which edges we want +to expand. If empty, all edges are valid. If not empty, only edges with one of +the given types are valid.") + (existing-node :bool :documentation "If the given node atom refer to a symbol +that has already been expanded and should be just validated in the frame.")) + (:serialize (:slk))) + +(lcp:define-class expand (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (input-symbol "Symbol" :scope :public) + (common "ExpandCommon" :scope :public) + (view "::storage::v3::View" :scope :public + :documentation + "State from which the input node should get expanded.")) + (:documentation + "Expansion operator. For a node existing in the frame it +expands one edge and one node and places them on the frame. + +This class does not handle node/edge filtering based on +properties, labels and edge types. However, it does handle +filtering on existing node / edge. + +Filtering on existing means that for a pattern that references +an already declared node or edge (for example in +MATCH (a) MATCH (a)--(b)), +only expansions that match defined equalities are successfully +pulled.") + (:public + #>cpp + /** + * Creates an expansion. All parameters except input and input_symbol are + * forwarded to @c ExpandCommon and are documented there. + * + * @param input Optional logical operator that preceeds this one. + * @param input_symbol Symbol that points to a VertexAccessor in the frame + * that expansion should emanate from. + */ + Expand(const std::shared_ptr &input, Symbol input_symbol, + Symbol node_symbol, Symbol edge_symbol, EdgeAtom::Direction direction, + const std::vector &edge_types, bool existing_node, + storage::v3::View view); + + Expand() {} + + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + + class ExpandCursor : public Cursor { + public: + ExpandCursor(const Expand &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + using InEdgeT = std::remove_reference_t().InEdges(storage::v3::View::OLD))>; + using InEdgeIteratorT = decltype(std::declval().begin()); + using OutEdgeT = std::remove_reference_t().OutEdges(storage::v3::View::OLD))>; + using OutEdgeIteratorT = decltype(std::declval().begin()); + + const Expand &self_; + const UniqueCursorPtr input_cursor_; + + // The iterable over edges and the current edge iterator are referenced via + // optional because they can not be initialized in the constructor of + // this class. They are initialized once for each pull from the input. + std::optional in_edges_; + std::optional in_edges_it_; + std::optional out_edges_; + std::optional out_edges_it_; + + bool InitEdges(Frame &, ExecutionContext &); + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-struct expansion-lambda () + ((inner-edge-symbol "Symbol" :documentation "Currently expanded edge symbol.") + (inner-node-symbol "Symbol" :documentation "Currently expanded node symbol.") + (expression "Expression *" :documentation "Expression used in lambda during expansion." + :slk-save #'slk-save-ast-pointer + :slk-load (lambda (member) + #>cpp + self->${member} = query::v2::LoadAstPointer( + ast_storage, reader); + cpp<#))) + (:serialize (:slk :load-args '((ast-storage "query::v2::AstStorage *")))) + (:clone :args '((storage "AstStorage *")))) + +(lcp:define-class expand-variable (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (input-symbol "Symbol" :scope :public) + (common "ExpandCommon" :scope :public) + (type "::EdgeAtom::Type" :scope :public) + (is-reverse :bool :scope :public :documentation + "True if the path should be written as expanding from node_symbol to input_symbol.") + (lower-bound "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Optional lower bound of the variable length expansion, defaults are (1, inf)") + (upper-bound "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression") + :documentation "Optional upper bound of the variable length expansion, defaults are (1, inf)") + (filter-lambda "ExpansionLambda" + :scope :public + :slk-load (lambda (member) + #>cpp + slk::Load(&self->${member}, reader, &helper->ast_storage); + cpp<#)) + (weight-lambda "std::optional" :scope :public + :slk-load (lambda (member) + #>cpp + bool has_value; + slk::Load(&has_value, reader); + if (!has_value) { + self->${member} = std::nullopt; + return; + } + query::v2::plan::ExpansionLambda lambda; + slk::Load(&lambda, reader, &helper->ast_storage); + self->${member}.emplace(lambda); + cpp<#)) + (total-weight "std::optional" :scope :public)) + (:documentation + "Variable-length expansion operator. For a node existing in +the frame it expands a variable number of edges and places them +(in a list-type TypedValue), as well as the final destination node, +on the frame. + +This class does not handle node/edge filtering based on +properties, labels and edge types. However, it does handle +filtering on existing node / edge. Additionally it handles's +edge-uniquess (cyphermorphism) because it's not feasable to do +later. + +Filtering on existing means that for a pattern that references +an already declared node or edge (for example in +MATCH (a) MATCH (a)--(b)), +only expansions that match defined equalities are succesfully +pulled.") + (:public + #>cpp + ExpandVariable() {} + + /** + * Creates a variable-length expansion. Most params are forwarded + * to the @c ExpandCommon constructor, and are documented there. + * + * Expansion length bounds are both inclusive (as in Neo's Cypher + * implementation). + * + * @param input Optional logical operator that preceeds this one. + * @param input_symbol Symbol that points to a VertexAccessor in the frame + * that expansion should emanate from. + * @param type - Either Type::DEPTH_FIRST (default variable-length expansion), + * or Type::BREADTH_FIRST. + * @param is_reverse Set to `true` if the edges written on frame should expand + * from `node_symbol` to `input_symbol`. Opposed to the usual expanding + * from `input_symbol` to `node_symbol`. + * @param lower_bound An optional indicator of the minimum number of edges + * that get expanded (inclusive). + * @param upper_bound An optional indicator of the maximum number of edges + * that get expanded (inclusive). + * @param inner_edge_symbol Like `inner_node_symbol` + * @param inner_node_symbol For each expansion the node expanded into is + * assigned to this symbol so it can be evaulated by the 'where' + * expression. + * @param filter_ The filter that must be satisfied for an expansion to + * succeed. Can use inner(node/edge) symbols. If nullptr, it is ignored. + */ + ExpandVariable(const std::shared_ptr &input, + Symbol input_symbol, Symbol node_symbol, Symbol edge_symbol, + EdgeAtom::Type type, EdgeAtom::Direction direction, + const std::vector &edge_types, + bool is_reverse, Expression *lower_bound, + Expression *upper_bound, bool existing_node, + ExpansionLambda filter_lambda, + std::optional weight_lambda, + std::optional total_weight); + + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + // the Cursors are not declared in the header because + // it's edges_ and edges_it_ are decltyped using a helper function + // that should be inaccessible (private class function won't compile) + friend class ExpandVariableCursor; + friend class ExpandWeightedShortestPathCursor; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class construct-named-path (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (path-symbol "Symbol" :scope :public) + (path-elements "std::vector" :scope :public)) + (:documentation + "Constructs a named path from its elements and places it on the frame.") + (:public + #>cpp + ConstructNamedPath() {} + ConstructNamedPath(const std::shared_ptr &input, + Symbol path_symbol, + const std::vector &path_elements) + : input_(input), + path_symbol_(path_symbol), + path_elements_(path_elements) {} + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class filter (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:documentation + "Filter whose Pull returns true only when the given expression +evaluates into true. + +The given expression is assumed to return either NULL (treated as false) or +a boolean value.") + (:public + #>cpp + Filter() {} + + Filter(const std::shared_ptr &input_, + Expression *expression_); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class FilterCursor : public Cursor { + public: + FilterCursor(const Filter &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const Filter &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class produce (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (named-expressions "std::vector" :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "NamedExpression"))) + (:documentation + "A logical operator that places an arbitrary number +of named expressions on the frame (the logical operator +for the RETURN clause). + +Supports optional input. When the input is provided, +it is Pulled from and the Produce succeeds once for +every input Pull (typically a MATCH/RETURN query). +When the input is not provided (typically a standalone +RETURN clause) the Produce's pull succeeds exactly once.") + (:public + #>cpp + Produce() {} + + Produce(const std::shared_ptr &input, + const std::vector &named_expressions); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class ProduceCursor : public Cursor { + public: + ProduceCursor(const Produce &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const Produce &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class delete (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (expressions "std::vector" :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression")) + (detach :bool :scope :public :documentation + "Whether the vertex should be detached before deletion. If not detached, + and has connections, an error is raised when deleting edges.")) + (:documentation + "Operator for deleting vertices and edges. + +Has a flag for using DETACH DELETE when deleting vertices.") + (:public + #>cpp + Delete() {} + + Delete(const std::shared_ptr &input_, + const std::vector &expressions, bool detach_); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class DeleteCursor : public Cursor { + public: + DeleteCursor(const Delete &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const Delete &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class set-property (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (property "::storage::v3::PropertyId" :scope :public) + (lhs "PropertyLookup *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "PropertyLookup")) + (rhs "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:documentation + "Logical operator for setting a single property on a single vertex or edge. + +The property value is an expression that must evaluate to some type that +can be stored (a TypedValue that can be converted to PropertyValue).") + (:public + #>cpp + SetProperty() {} + + SetProperty(const std::shared_ptr &input, + storage::v3::PropertyId property, PropertyLookup *lhs, + Expression *rhs); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class SetPropertyCursor : public Cursor { + public: + SetPropertyCursor(const SetProperty &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const SetProperty &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class set-properties (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (input-symbol "Symbol" :scope :public) + (rhs "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (op "Op" :scope :public)) + (:documentation + "Logical operator for setting the whole property set on a vertex or an edge. + +The value being set is an expression that must evaluate to a vertex, edge or +map (literal or parameter). + +Supports setting (replacing the whole properties set with another) and +updating.") + (:public + (lcp:define-enum op + (update replace) + (:documentation "Defines how setting the properties works. + +@c UPDATE means that the current property set is augmented with additional +ones (existing props of the same name are replaced), while @c REPLACE means +that the old properties are discarded and replaced with new ones.") + (:serialize)) + + #>cpp + SetProperties() {} + + SetProperties(const std::shared_ptr &input, + Symbol input_symbol, Expression *rhs, Op op); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class SetPropertiesCursor : public Cursor { + public: + SetPropertiesCursor(const SetProperties &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const SetProperties &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class set-labels (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (input-symbol "Symbol" :scope :public) + (labels "std::vector" :scope :public)) + (:documentation + "Logical operator for setting an arbitrary number of labels on a Vertex. + +It does NOT remove labels that are already set on that Vertex.") + (:public + #>cpp + SetLabels() {} + + SetLabels(const std::shared_ptr &input, Symbol input_symbol, + const std::vector &labels); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class SetLabelsCursor : public Cursor { + public: + SetLabelsCursor(const SetLabels &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const SetLabels &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class remove-property (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (property "::storage::v3::PropertyId" :scope :public) + (lhs "PropertyLookup *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "PropertyLookup"))) + (:documentation + "Logical operator for removing a property from an edge or a vertex.") + (:public + #>cpp + RemoveProperty() {} + + RemoveProperty(const std::shared_ptr &input, + storage::v3::PropertyId property, PropertyLookup *lhs); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class RemovePropertyCursor : public Cursor { + public: + RemovePropertyCursor(const RemoveProperty &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const RemoveProperty &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class remove-labels (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (input-symbol "Symbol" :scope :public) + (labels "std::vector" :scope :public)) + (:documentation + "Logical operator for removing an arbitrary number of labels on a Vertex. + +If a label does not exist on a Vertex, nothing happens.") + (:public + #>cpp + RemoveLabels() {} + + RemoveLabels(const std::shared_ptr &input, + Symbol input_symbol, const std::vector &labels); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class RemoveLabelsCursor : public Cursor { + public: + RemoveLabelsCursor(const RemoveLabels &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const RemoveLabels &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class edge-uniqueness-filter (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (expand-symbol "Symbol" :scope :public) + (previous-symbols "std::vector" :scope :public)) + (:documentation + "Filter whose Pull returns true only when the given expand_symbol frame +value (the latest expansion) is not equal to any of the previous_symbols frame +values. + +Used for implementing Cyphermorphism. +Isomorphism is vertex-uniqueness. It means that two different vertices in a +pattern can not map to the same data vertex. +Cyphermorphism is edge-uniqueness (the above explanation applies). By default +Neo4j uses Cyphermorphism (that's where the name stems from, it is not a valid +graph-theory term). + +Supports variable-length-edges (uniqueness comparisons between edges and an +edge lists).") + (:public + #>cpp + EdgeUniquenessFilter() {} + + EdgeUniquenessFilter(const std::shared_ptr &input, + Symbol expand_symbol, + const std::vector &previous_symbols); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class EdgeUniquenessFilterCursor : public Cursor { + public: + EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &, + utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const EdgeUniquenessFilter &self_; + const UniqueCursorPtr input_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class accumulate (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (symbols "std::vector" :scope :public) + (advance-command :bool :scope :public)) + (:documentation + "Pulls everything from the input before passing it through. +Optionally advances the command after accumulation and before emitting. + +On the first Pull from this operator's Cursor the input Cursor will be Pulled +until it is empty. The results will be accumulated in the temporary cache. Once +the input Cursor is empty, this operator's Cursor will start returning cached +stuff from its Pull. + +This technique is used for ensuring all the operations from the +previous logical operator have been performed before exposing data +to the next. A typical use case is a `MATCH--SET--RETURN` +query in which every SET iteration must be performed before +RETURN starts iterating (see Memgraph Wiki for detailed reasoning). + +IMPORTANT: This operator does not cache all the results but only those +elements from the frame whose symbols (frame positions) it was given. +All other frame positions will contain undefined junk after this +operator has executed, and should not be used. + +This operator can also advance the command after the accumulation and +before emitting. If the command gets advanced, every value that +has been cached will be reconstructed before Pull returns. + +@param input Input @c LogicalOperator. +@param symbols A vector of Symbols that need to be accumulated + and exposed to the next op.") + (:public + #>cpp + Accumulate() {} + + Accumulate(const std::shared_ptr &input, + const std::vector &symbols, bool advance_command = false); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class aggregate (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (aggregations "std::vector" :scope :public + :slk-save (lambda (member) + #>cpp + size_t size = self.${member}.size(); + slk::Save(size, builder); + for (const auto &v : self.${member}) { + slk::Save(v, builder, helper); + } + cpp<#) + :slk-load (lambda (member) + #>cpp + size_t size; + slk::Load(&size, reader); + self->${member}.resize(size); + for (size_t i = 0; + i < size; + ++i) { + slk::Load(&self->${member}[i], reader, helper); + } + cpp<#)) + (group-by "std::vector" :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression")) + (remember "std::vector" :scope :public)) + (:documentation + "Performs an arbitrary number of aggregations of data +from the given input grouped by the given criteria. + +Aggregations are defined by triples that define +(input data expression, type of aggregation, output symbol). +Input data is grouped based on the given set of named +expressions. Grouping is done on unique values. + +IMPORTANT: +Operators taking their input from an aggregation are only +allowed to use frame values that are either aggregation +outputs or group-by named-expressions. All other frame +elements are in an undefined state after aggregation.") + (:public + (lcp:define-struct element () + ((value "Expression *" + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (key "Expression *" + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (op "::Aggregation::Op") + (output-sym "Symbol")) + (:documentation + "An aggregation element, contains: + (input data expression, key expression - only used in COLLECT_MAP, type of + aggregation, output symbol).") + (:serialize (:slk :save-args '((helper "query::v2::plan::LogicalOperator::SaveHelper *")) + :load-args '((helper "query::v2::plan::LogicalOperator::SlkLoadHelper *")))) + (:clone :args '((storage "AstStorage *")))) + #>cpp + Aggregate() = default; + Aggregate(const std::shared_ptr &input, + const std::vector &aggregations, + const std::vector &group_by, + const std::vector &remember); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class skip (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:documentation + "Skips a number of Pulls from the input op. + +The given expression determines how many Pulls from the input +should be skipped (ignored). +All other successful Pulls from the +input are simply passed through. + +The given expression is evaluated after the first Pull from +the input, and only once. Neo does not allow this expression +to contain identifiers, and neither does Memgraph, but this +operator's implementation does not expect this.") + (:public + #>cpp + Skip() {} + + Skip(const std::shared_ptr &input, Expression *expression); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class SkipCursor : public Cursor { + public: + SkipCursor(const Skip &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const Skip &self_; + const UniqueCursorPtr input_cursor_; + // init to_skip_ to -1, indicating + // that it's still unknown (input has not been Pulled yet) + int64_t to_skip_{-1}; + int64_t skipped_{0}; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class limit (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression"))) + (:documentation + "Limits the number of Pulls from the input op. + +The given expression determines how many +input Pulls should be passed through. The input is not +Pulled once this limit is reached. Note that this has +implications: the out-of-bounds input Pulls are never +evaluated. + +The limit expression must NOT use anything from the +Frame. It is evaluated before the first Pull from the +input. This is consistent with Neo (they don't allow +identifiers in limit expressions), and it's necessary +when limit evaluates to 0 (because 0 Pulls from the +input should be performed).") + (:public + #>cpp + Limit() {} + + Limit(const std::shared_ptr &input, Expression *expression); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class LimitCursor : public Cursor { + public: + LimitCursor(const Limit &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const Limit &self_; + UniqueCursorPtr input_cursor_; + // init limit_ to -1, indicating + // that it's still unknown (Cursor has not been Pulled yet) + int64_t limit_{-1}; + int64_t pulled_{0}; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class order-by (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (compare "TypedValueVectorCompare" :scope :public) + (order-by "std::vector" :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression")) + (output-symbols "std::vector" :scope :public)) + (:documentation + "Logical operator for ordering (sorting) results. + +Sorts the input rows based on an arbitrary number of +Expressions. Ascending or descending ordering can be chosen +for each independently (not providing enough orderings +results in a runtime error). + +For each row an arbitrary number of Frame elements can be +remembered. Only these elements (defined by their Symbols) +are valid for usage after the OrderBy operator.") + (:public + #>cpp + OrderBy() {} + + OrderBy(const std::shared_ptr &input, + const std::vector &order_by, + const std::vector &output_symbols); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class merge (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (merge-match "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (merge-create "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer)) + (:documentation + "Merge operator. For every sucessful Pull from the +input operator a Pull from the merge_match is attempted. All +successfull Pulls from the merge_match are passed on as output. +If merge_match Pull does not yield any elements, a single Pull +from the merge_create op is performed. + +The input logical op is optional. If false (nullptr) +it will be replaced by a Once op. + +For an argumentation of this implementation see the wiki +documentation.") + (:public + #>cpp + Merge() {} + + Merge(const std::shared_ptr &input, + const std::shared_ptr &merge_match, + const std::shared_ptr &merge_create); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + // TODO: Consider whether we want to treat Merge as having single input. It + // makes sense that we do, because other branches are executed depending on + // the input. + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class MergeCursor : public Cursor { + public: + MergeCursor(const Merge &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const UniqueCursorPtr input_cursor_; + const UniqueCursorPtr merge_match_cursor_; + const UniqueCursorPtr merge_create_cursor_; + + // indicates if the next Pull from this cursor + // should perform a pull from input_cursor_ + // this is true when: + // - first Pulling from this cursor + // - previous Pull from this cursor exhausted the merge_match_cursor + bool pull_input_{true}; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class optional (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (optional "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (optional-symbols "std::vector" :scope :public)) + (:documentation + "Optional operator. Used for optional match. For every +successful Pull from the input branch a Pull from the optional +branch is attempted (and Pulled from till exhausted). If zero +Pulls succeed from the optional branch, the Optional operator +sets the optional symbols to TypedValue::Null on the Frame +and returns true, once.") + (:public + #>cpp + Optional() {} + + Optional(const std::shared_ptr &input, + const std::shared_ptr &optional, + const std::vector &optional_symbols); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:private + #>cpp + class OptionalCursor : public Cursor { + public: + OptionalCursor(const Optional &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const Optional &self_; + const UniqueCursorPtr input_cursor_; + const UniqueCursorPtr optional_cursor_; + // indicates if the next Pull from this cursor should + // perform a Pull from the input_cursor_ + // this is true when: + // - first pulling from this Cursor + // - previous Pull from this cursor exhausted the optional_cursor_ + bool pull_input_{true}; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class unwind (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (input-expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (output-symbol "Symbol" :scope :public)) + (:documentation + "Takes a list TypedValue as it's input and yields each +element as it's output. + +Input is optional (unwind can be the first clause in a query).") + (:public + #>cpp + Unwind() {} + + Unwind(const std::shared_ptr &input, + Expression *input_expression_, Symbol output_symbol); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { + return true; } + std::shared_ptr input() const override { + return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class distinct (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (value-symbols "std::vector" :scope :public)) + (:documentation + "Ensures that only distinct rows are yielded. +This implementation accepts a vector of Symbols +which define a row. Only those Symbols are valid +for use in operators following Distinct. + +This implementation maintains input ordering.") + (:public + #>cpp + Distinct() {} + + Distinct(const std::shared_ptr &input, + const std::vector &value_symbols); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class union (logical-operator) + ((left-op "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (right-op "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (union-symbols "std::vector" :scope :public) + (left-symbols "std::vector" :scope :public) + (right-symbols "std::vector" :scope :public)) + (:documentation + "A logical operator that applies UNION operator on inputs and places the +result on the frame. + +This operator takes two inputs, a vector of symbols for the result, and vectors +of symbols used by each of the inputs.") + (:public + #>cpp + Union() {} + + Union(const std::shared_ptr &left_op, + const std::shared_ptr &right_op, + const std::vector &union_symbols, + const std::vector &left_symbols, + const std::vector &right_symbols); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override; + std::shared_ptr input() const override; + void set_input(std::shared_ptr) override; + cpp<#) + (:private + #>cpp + class UnionCursor : public Cursor { + public: + UnionCursor(const Union &, utils::MemoryResource *); + bool Pull(Frame &, ExecutionContext &) override; + void Shutdown() override; + void Reset() override; + + private: + const Union &self_; + const UniqueCursorPtr left_cursor_, right_cursor_; + }; + cpp<#) + (:serialize (:slk)) + (:clone)) + +;; TODO: We should probably output this operator in regular planner, not just +;; distributed planner. +(lcp:define-class cartesian (logical-operator) + ((left-op "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (left-symbols "std::vector" :scope :public) + (right-op "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (right-symbols "std::vector" :scope :public)) + (:documentation + "Operator for producing a Cartesian product from 2 input branches") + (:public + #>cpp + Cartesian() {} + /** Construct the operator with left input branch and right input branch. */ + Cartesian(const std::shared_ptr &left_op, + const std::vector &left_symbols, + const std::shared_ptr &right_op, + const std::vector &right_symbols) + : left_op_(left_op), + left_symbols_(left_symbols), + right_op_(right_op), + right_symbols_(right_symbols) {} + + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override; + std::shared_ptr input() const override; + void set_input(std::shared_ptr) override; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class output-table (logical-operator) + ((output-symbols "std::vector" :scope :public :dont-save t) + (callback "std::function>(Frame *, ExecutionContext *)>" + :scope :public :dont-save t :clone :copy)) + (:documentation "An operator that outputs a table, producing a single row on each pull") + (:public + #>cpp + OutputTable() {} + OutputTable( + std::vector output_symbols, + std::function>(Frame *, ExecutionContext *)> + callback); + OutputTable(std::vector output_symbols, + std::vector> rows); + + bool Accept(HierarchicalLogicalOperatorVisitor &) override { + LOG_FATAL("OutputTable operator should not be visited!"); + } + + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override { + return output_symbols_; + } + std::vector ModifiedSymbols(const SymbolTable &) const override { + return output_symbols_; + } + + bool HasSingleInput() const override; + std::shared_ptr input() const override; + void set_input(std::shared_ptr input) override; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class output-table-stream (logical-operator) + ((output-symbols "std::vector" :scope :public :dont-save t) + (callback "std::function>(Frame *, ExecutionContext *)>" + :scope :public :dont-save t :clone :copy)) + (:documentation "An operator that outputs a table, producing a single row on each pull. +This class is different from @c OutputTable in that its callback doesn't fetch all rows +at once. Instead, each call of the callback should return a single row of the table.") + (:public + #>cpp + OutputTableStream() {} + OutputTableStream( + std::vector output_symbols, + std::function>(Frame *, ExecutionContext *)> + callback); + + bool Accept(HierarchicalLogicalOperatorVisitor &) override { + LOG_FATAL("OutputTableStream operator should not be visited!"); + } + + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override { + return output_symbols_; + } + std::vector ModifiedSymbols(const SymbolTable &) const override { + return output_symbols_; + } + + bool HasSingleInput() const override; + std::shared_ptr input() const override; + void set_input(std::shared_ptr input) override; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class call-procedure (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (procedure-name "std::string" :scope :public) + (arguments "std::vector" + :scope :public + :slk-save #'slk-save-ast-vector + :slk-load (slk-load-ast-vector "Expression")) + (result-fields "std::vector" :scope :public) + (result-symbols "std::vector" :scope :public) + (memory-limit "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (memory-scale "size_t" :initval "1024U" :scope :public) + (is_write :bool :scope :public)) + (:public + #>cpp + CallProcedure() = default; + CallProcedure(std::shared_ptr input, std::string name, + std::vector arguments, + std::vector fields, std::vector symbols, + Expression *memory_limit, size_t memory_scale, bool is_write); + + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + + static void IncrementCounter(const std::string &procedure_name); + static std::unordered_map GetAndResetCounters(); + cpp<#) + (:private + #>cpp + inline static utils::Synchronized, utils::SpinLock> procedure_counters_; + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class load-csv (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (file "Expression *" :scope :public) + (with_header "bool" :scope :public) + (ignore_bad "bool" :scope :public) + (delimiter "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (quote "Expression *" :initval "nullptr" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (row_var "Symbol" :scope :public)) + (:public + #>cpp + LoadCsv() = default; + LoadCsv(std::shared_ptr input, Expression *file, bool with_header, bool ignore_bad, + Expression* delimiter, Expression* quote, Symbol row_var); + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector OutputSymbols(const SymbolTable &) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = input; + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:define-class foreach (logical-operator) + ((input "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (update-clauses "std::shared_ptr" :scope :public + :slk-save #'slk-save-operator-pointer + :slk-load #'slk-load-operator-pointer) + (expression "Expression *" :scope :public + :slk-save #'slk-save-ast-pointer + :slk-load (slk-load-ast-pointer "Expression")) + (loop-variable-symbol "Symbol" :scope :public)) + + (:documentation + "Iterates over a collection of elements and applies one or more update +clauses. +") + (:public + #>cpp + Foreach() = default; + Foreach(std::shared_ptr input, + std::shared_ptr updates, + Expression *named_expr, + Symbol loop_variable_symbol); + + bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; + UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; + std::vector ModifiedSymbols(const SymbolTable &) const override; + bool HasSingleInput() const override { return true; } + std::shared_ptr input() const override { return input_; } + void set_input(std::shared_ptr input) override { + input_ = std::move(input); + } + cpp<#) + (:serialize (:slk)) + (:clone)) + +(lcp:pop-namespace) ;; plan +(lcp:pop-namespace) ;; v2 +(lcp:pop-namespace) ;; query +(lcp:pop-namespace) ;; memgraph diff --git a/src/query/v2/plan/planner.hpp b/src/query/v2/plan/planner.hpp new file mode 100644 index 000000000..fe9e88f32 --- /dev/null +++ b/src/query/v2/plan/planner.hpp @@ -0,0 +1,158 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +/// @file +/// This file is an entry point for invoking various planners via the following +/// API: +/// * `MakeLogicalPlanForSingleQuery` +/// * `MakeLogicalPlan` + +#pragma once + +#include "query/v2/plan/cost_estimator.hpp" +#include "query/v2/plan/operator.hpp" +#include "query/v2/plan/preprocess.hpp" +#include "query/v2/plan/pretty_print.hpp" +#include "query/v2/plan/rewrite/index_lookup.hpp" +#include "query/v2/plan/rule_based_planner.hpp" +#include "query/v2/plan/variable_start_planner.hpp" +#include "query/v2/plan/vertex_count_cache.hpp" + +namespace memgraph::query::v2 { + +class AstStorage; +class SymbolTable; + +namespace plan { + +class PostProcessor final { + Parameters parameters_; + + public: + using ProcessedPlan = std::unique_ptr; + + explicit PostProcessor(const Parameters ¶meters) : parameters_(parameters) {} + + template + std::unique_ptr Rewrite(std::unique_ptr plan, TPlanningContext *context) { + return RewriteWithIndexLookup(std::move(plan), context->symbol_table, context->ast_storage, context->db); + } + + template + double EstimatePlanCost(const std::unique_ptr &plan, TVertexCounts *vertex_counts) { + return query::v2::plan::EstimatePlanCost(vertex_counts, parameters_, *plan); + } + + template + std::unique_ptr MergeWithCombinator(std::unique_ptr curr_op, + std::unique_ptr last_op, const Tree &combinator, + TPlanningContext *context) { + if (const auto *union_ = utils::Downcast(&combinator)) { + return std::unique_ptr( + impl::GenUnion(*union_, std::move(last_op), std::move(curr_op), *context->symbol_table)); + } + throw utils::NotYetImplemented("query combinator"); + } + + template + std::unique_ptr MakeDistinct(std::unique_ptr last_op, TPlanningContext *context) { + auto output_symbols = last_op->OutputSymbols(*context->symbol_table); + return std::make_unique(std::move(last_op), output_symbols); + } +}; + +/// @brief Generates the LogicalOperator tree for a single query and returns the +/// resulting plan. +/// +/// @tparam TPlanner Type of the planner used for generation. +/// @tparam TDbAccessor Type of the database accessor used for generation. +/// @param vector of @c SingleQueryPart from the single query +/// @param context PlanningContext used for generating plans. +/// @return @c PlanResult which depends on the @c TPlanner used. +/// +/// @sa PlanningContext +/// @sa RuleBasedPlanner +/// @sa VariableStartPlanner +template